<!DOCTYPE html>
<!-- saved from url=(0042)https://zybuluo.com/hanbingtao/note/541458 -->
<html class="theme theme-white"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    
    
    <meta name="description" content="Cmd Markdown 编辑阅读器，支持实时同步预览，区分写作和阅读模式，支持在线存储，分享文稿网址。">
    <meta name="author" content="Jiawei Zhang">

    <meta name="viewport" content="width=device-width, initial-scale=1.0">

    
    <title>零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器</title>


    <link href="https://zybuluo.com/static/img/favicon.png" type="image/x-icon" rel="icon">

    <link href="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/1bc053c8.base.lib.min.css" rel="stylesheet" media="screen">


    
    <!-- id="prettify-style" will be used to get the link element below and change href to change prettify code, so it can't be in beginmin/endmin block. -->
    <link id="prettify-style" href="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/prettify-cmd.css" type="text/css" rel="stylesheet">
    <!--
    <link id="mermaid-style" href="https://zybuluo.com/static/editor/libs/mermaid/mermaid.forest.css" type="text/css" rel="stylesheet">
    -->
    <link href="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/45c7d56d.layout.min.css" rel="stylesheet" media="screen">


    

    <script async="" src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/analytics.js"></script><script>
      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

      ga('create', 'UA-44461741-1', 'zybuluo.com');
      ga('send', 'pageview');
    </script>
<style type="text/css">.MathJax_Hover_Frame {border-radius: .25em; -webkit-border-radius: .25em; -moz-border-radius: .25em; -khtml-border-radius: .25em; box-shadow: 0px 0px 15px #83A; -webkit-box-shadow: 0px 0px 15px #83A; -moz-box-shadow: 0px 0px 15px #83A; -khtml-box-shadow: 0px 0px 15px #83A; border: 1px solid #A6D ! important; display: inline-block; position: absolute}
.MathJax_Hover_Arrow {position: absolute; width: 15px; height: 11px; cursor: pointer}
</style><style type="text/css">#MathJax_About {position: fixed; left: 50%; width: auto; text-align: center; border: 3px outset; padding: 1em 2em; background-color: #DDDDDD; color: black; cursor: default; font-family: message-box; font-size: 120%; font-style: normal; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 15px; -webkit-border-radius: 15px; -moz-border-radius: 15px; -khtml-border-radius: 15px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
.MathJax_Menu {position: absolute; background-color: white; color: black; width: auto; padding: 5px 0px; border: 1px solid #CCCCCC; margin: 0; cursor: default; font: menu; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 5px; -webkit-border-radius: 5px; -moz-border-radius: 5px; -khtml-border-radius: 5px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
.MathJax_MenuItem {padding: 1px 2em; background: transparent}
.MathJax_MenuArrow {position: absolute; right: .5em; color: #666666}
.MathJax_MenuActive .MathJax_MenuArrow {color: white}
.MathJax_MenuCheck {position: absolute; left: .7em}
.MathJax_MenuRadioCheck {position: absolute; left: .7em}
.MathJax_MenuLabel {padding: 1px 2em 3px 1.33em; font-style: italic}
.MathJax_MenuRule {border-top: 1px solid #DDDDDD; margin: 4px 3px}
.MathJax_MenuDisabled {color: GrayText}
.MathJax_MenuActive {background-color: #606872; color: white}
.MathJax_Menu_Close {position: absolute; width: 31px; height: 31px; top: -15px; left: -15px}
</style><style type="text/css">#MathJax_Zoom {position: absolute; background-color: #F0F0F0; overflow: auto; display: block; z-index: 301; padding: .5em; border: 1px solid black; margin: 0; font-weight: normal; font-style: normal; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; box-shadow: 5px 5px 15px #AAAAAA; -webkit-box-shadow: 5px 5px 15px #AAAAAA; -moz-box-shadow: 5px 5px 15px #AAAAAA; -khtml-box-shadow: 5px 5px 15px #AAAAAA; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
#MathJax_ZoomOverlay {position: absolute; left: 0; top: 0; z-index: 300; display: inline-block; width: 100%; height: 100%; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
#MathJax_ZoomFrame {position: relative; display: inline-block; height: 0; width: 0}
#MathJax_ZoomEventTrap {position: absolute; left: 0; top: 0; z-index: 302; display: inline-block; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
</style><style type="text/css">.MathJax_Preview {color: #888}
#MathJax_Message {position: fixed; left: 1em; bottom: 1.5em; background-color: #E6E6E6; border: 1px solid #959595; margin: 0px; padding: 2px 8px; z-index: 102; color: black; font-size: 80%; width: auto; white-space: nowrap}
#MathJax_MSIE_Frame {position: absolute; top: 0; left: 0; width: 0px; z-index: 101; border: 0px; margin: 0px; padding: 0px}
.MathJax_Error {color: #CC0000; font-style: italic}
</style><style type="text/css">.MathJax_SVG_Display {text-align: center; margin: 1em 0em; position: relative; display: block; width: 100%}
#MathJax_SVG_Tooltip {background-color: InfoBackground; color: InfoText; border: 1px solid black; box-shadow: 2px 2px 5px #AAAAAA; -webkit-box-shadow: 2px 2px 5px #AAAAAA; -moz-box-shadow: 2px 2px 5px #AAAAAA; -khtml-box-shadow: 2px 2px 5px #AAAAAA; padding: 3px 4px; z-index: 401; position: absolute; left: 0; top: 0; width: auto; height: auto; display: none}
.MathJax_SVG {display: inline; font-style: normal; font-weight: normal; line-height: normal; font-size: 100%; font-size-adjust: none; text-indent: 0; text-align: left; text-transform: none; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; direction: ltr; border: 0; padding: 0; margin: 0}
.MathJax_SVG * {transition: none; -webkit-transition: none; -moz-transition: none; -ms-transition: none; -o-transition: none}
.mjx-svg-href {fill: blue; stroke: blue}
.MathJax_SVG_Processing {visibility: hidden; position: absolute; top: 0; left: 0; width: 0; height: 0; overflow: hidden; display: block}
.MathJax_SVG_Processed {display: none!important}
.MathJax_SVG_ExBox {display: block; overflow: hidden; width: 1px; height: 60ex}
.MathJax_SVG .noError {vertical-align: ; font-size: 90%; text-align: left; color: black; padding: 1px 3px; border: 1px solid}
</style></head>

<body class="theme  theme-white pace-done"><div style="visibility: hidden; overflow: hidden; position: absolute; top: 0px; height: 1px; width: auto; padding: 0px; border: 0px; margin: 0px; text-align: left; text-indent: 0px; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal;"><div id="MathJax_SVG_Hidden"></div><svg><defs id="MathJax_SVG_glyphs"><path id="MJMAIN-78" stroke-width="1" d="M201 0Q189 3 102 3Q26 3 17 0H11V46H25Q48 47 67 52T96 61T121 78T139 96T160 122T180 150L226 210L168 288Q159 301 149 315T133 336T122 351T113 363T107 370T100 376T94 379T88 381T80 383Q74 383 44 385H16V431H23Q59 429 126 429Q219 429 229 431H237V385Q201 381 201 369Q201 367 211 353T239 315T268 274L272 270L297 304Q329 345 329 358Q329 364 327 369T322 376T317 380T310 384L307 385H302V431H309Q324 428 408 428Q487 428 493 431H499V385H492Q443 385 411 368Q394 360 377 341T312 257L296 236L358 151Q424 61 429 57T446 50Q464 46 499 46H516V0H510H502Q494 1 482 1T457 2T432 2T414 3Q403 3 377 3T327 1L304 0H295V46H298Q309 46 320 51T331 63Q331 65 291 120L250 175Q249 174 219 133T185 88Q181 83 181 74Q181 63 188 55T206 46Q208 46 208 23V0H201Z"></path><path id="MJMATHI-74" stroke-width="1" d="M26 385Q19 392 19 395Q19 399 22 411T27 425Q29 430 36 430T87 431H140L159 511Q162 522 166 540T173 566T179 586T187 603T197 615T211 624T229 626Q247 625 254 615T261 596Q261 589 252 549T232 470L222 433Q222 431 272 431H323Q330 424 330 420Q330 398 317 385H210L174 240Q135 80 135 68Q135 26 162 26Q197 26 230 60T283 144Q285 150 288 151T303 153H307Q322 153 322 145Q322 142 319 133Q314 117 301 95T267 48T216 6T155 -11Q125 -11 98 4T59 56Q57 64 57 83V101L92 241Q127 382 128 383Q128 385 77 385H26Z"></path><path id="MJMAIN-73" stroke-width="1" d="M295 316Q295 356 268 385T190 414Q154 414 128 401Q98 382 98 349Q97 344 98 336T114 312T157 287Q175 282 201 278T245 269T277 256Q294 248 310 236T342 195T359 133Q359 71 321 31T198 -10H190Q138 -10 94 26L86 19L77 10Q71 4 65 -1L54 -11H46H42Q39 -11 33 -5V74V132Q33 153 35 157T45 162H54Q66 162 70 158T75 146T82 119T101 77Q136 26 198 26Q295 26 295 104Q295 133 277 151Q257 175 194 187T111 210Q75 227 54 256T33 318Q33 357 50 384T93 424T143 442T187 447H198Q238 447 268 432L283 424L292 431Q302 440 314 448H322H326Q329 448 335 442V310L329 304H301Q295 310 295 316Z"></path><path id="MJMAIN-6F" stroke-width="1" d="M28 214Q28 309 93 378T250 448Q340 448 405 380T471 215Q471 120 407 55T250 -10Q153 -10 91 57T28 214ZM250 30Q372 30 372 193V225V250Q372 272 371 288T364 326T348 362T317 390T268 410Q263 411 252 411Q222 411 195 399Q152 377 139 338T126 246V226Q126 130 145 91Q177 30 250 30Z"></path><path id="MJMAIN-2212" stroke-width="1" d="M84 237T84 250T98 270H679Q694 262 694 250T679 230H98Q84 237 84 250Z"></path><path id="MJMAIN-31" stroke-width="1" d="M213 578L200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641T273 663Q275 666 285 666Q294 666 302 660V361L303 61Q310 54 315 52T339 48T401 46H427V0H416Q395 3 257 3Q121 3 100 0H88V46H114Q136 46 152 46T177 47T193 50T201 52T207 57T213 61V578Z"></path><path id="MJMAIN-28" stroke-width="1" d="M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z"></path><path id="MJMAIN-29" stroke-width="1" d="M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z"></path><path id="MJMAIN-3D" stroke-width="1" d="M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z"></path><path id="MJMATHI-67" stroke-width="1" d="M311 43Q296 30 267 15T206 0Q143 0 105 45T66 160Q66 265 143 353T314 442Q361 442 401 394L404 398Q406 401 409 404T418 412T431 419T447 422Q461 422 470 413T480 394Q480 379 423 152T363 -80Q345 -134 286 -169T151 -205Q10 -205 10 -137Q10 -111 28 -91T74 -71Q89 -71 102 -80T116 -111Q116 -121 114 -130T107 -144T99 -154T92 -162L90 -164H91Q101 -167 151 -167Q189 -167 211 -155Q234 -144 254 -122T282 -75Q288 -56 298 -13Q311 35 311 43ZM384 328L380 339Q377 350 375 354T369 368T359 382T346 393T328 402T306 405Q262 405 221 352Q191 313 171 233T151 117Q151 38 213 38Q269 38 323 108L331 118L384 328Z"></path><path id="MJMATHI-56" stroke-width="1" d="M52 648Q52 670 65 683H76Q118 680 181 680Q299 680 320 683H330Q336 677 336 674T334 656Q329 641 325 637H304Q282 635 274 635Q245 630 242 620Q242 618 271 369T301 118L374 235Q447 352 520 471T595 594Q599 601 599 609Q599 633 555 637Q537 637 537 648Q537 649 539 661Q542 675 545 679T558 683Q560 683 570 683T604 682T668 681Q737 681 755 683H762Q769 676 769 672Q769 655 760 640Q757 637 743 637Q730 636 719 635T698 630T682 623T670 615T660 608T652 599T645 592L452 282Q272 -9 266 -16Q263 -18 259 -21L241 -22H234Q216 -22 216 -15Q213 -9 177 305Q139 623 138 626Q133 637 76 637H59Q52 642 52 648Z"></path><path id="MJMAIN-32" stroke-width="1" d="M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z"></path><path id="MJMATHI-66" stroke-width="1" d="M118 -162Q120 -162 124 -164T135 -167T147 -168Q160 -168 171 -155T187 -126Q197 -99 221 27T267 267T289 382V385H242Q195 385 192 387Q188 390 188 397L195 425Q197 430 203 430T250 431Q298 431 298 432Q298 434 307 482T319 540Q356 705 465 705Q502 703 526 683T550 630Q550 594 529 578T487 561Q443 561 443 603Q443 622 454 636T478 657L487 662Q471 668 457 668Q445 668 434 658T419 630Q412 601 403 552T387 469T380 433Q380 431 435 431Q480 431 487 430T498 424Q499 420 496 407T491 391Q489 386 482 386T428 385H372L349 263Q301 15 282 -47Q255 -132 212 -173Q175 -205 139 -205Q107 -205 81 -186T55 -132Q55 -95 76 -78T118 -61Q162 -61 162 -103Q162 -122 151 -136T127 -157L118 -162Z"></path><path id="MJMATHI-55" stroke-width="1" d="M107 637Q73 637 71 641Q70 643 70 649Q70 673 81 682Q83 683 98 683Q139 681 234 681Q268 681 297 681T342 682T362 682Q378 682 378 672Q378 670 376 658Q371 641 366 638H364Q362 638 359 638T352 638T343 637T334 637Q295 636 284 634T266 623Q265 621 238 518T184 302T154 169Q152 155 152 140Q152 86 183 55T269 24Q336 24 403 69T501 205L552 406Q599 598 599 606Q599 633 535 637Q511 637 511 648Q511 650 513 660Q517 676 519 679T529 683Q532 683 561 682T645 680Q696 680 723 681T752 682Q767 682 767 672Q767 650 759 642Q756 637 737 637Q666 633 648 597Q646 592 598 404Q557 235 548 205Q515 105 433 42T263 -22Q171 -22 116 34T60 167V183Q60 201 115 421Q164 622 164 628Q164 635 107 637Z"></path><path id="MJMAIN-2B" stroke-width="1" d="M56 237T56 250T70 270H369V420L370 570Q380 583 389 583Q402 583 409 568V270H707Q722 262 722 250T707 230H409V-68Q401 -82 391 -82H389H387Q375 -82 369 -68V230H70Q56 237 56 250Z"></path><path id="MJMATHI-57" stroke-width="1" d="M436 683Q450 683 486 682T553 680Q604 680 638 681T677 682Q695 682 695 674Q695 670 692 659Q687 641 683 639T661 637Q636 636 621 632T600 624T597 615Q597 603 613 377T629 138L631 141Q633 144 637 151T649 170T666 200T690 241T720 295T759 362Q863 546 877 572T892 604Q892 619 873 628T831 637Q817 637 817 647Q817 650 819 660Q823 676 825 679T839 682Q842 682 856 682T895 682T949 681Q1015 681 1034 683Q1048 683 1048 672Q1048 666 1045 655T1038 640T1028 637Q1006 637 988 631T958 617T939 600T927 584L923 578L754 282Q586 -14 585 -15Q579 -22 561 -22Q546 -22 542 -17Q539 -14 523 229T506 480L494 462Q472 425 366 239Q222 -13 220 -15T215 -19Q210 -22 197 -22Q178 -22 176 -15Q176 -12 154 304T131 622Q129 631 121 633T82 637H58Q51 644 51 648Q52 671 64 683H76Q118 680 176 680Q301 680 313 683H323Q329 677 329 674T327 656Q322 641 318 637H297Q236 634 232 620Q262 160 266 136L501 550L499 587Q496 629 489 632Q483 636 447 637Q428 637 422 639T416 648Q416 650 418 660Q419 664 420 669T421 676T424 680T428 682T436 683Z"></path><path id="MJMAIN-33" stroke-width="1" d="M127 463Q100 463 85 480T69 524Q69 579 117 622T233 665Q268 665 277 664Q351 652 390 611T430 522Q430 470 396 421T302 350L299 348Q299 347 308 345T337 336T375 315Q457 262 457 175Q457 96 395 37T238 -22Q158 -22 100 21T42 130Q42 158 60 175T105 193Q133 193 151 175T169 130Q169 119 166 110T159 94T148 82T136 74T126 70T118 67L114 66Q165 21 238 21Q293 21 321 74Q338 107 338 175V195Q338 290 274 322Q259 328 213 329L171 330L168 332Q166 335 166 348Q166 366 174 366Q202 366 232 371Q266 376 294 413T322 525V533Q322 590 287 612Q265 626 240 626Q208 626 181 615T143 592T132 580H135Q138 579 143 578T153 573T165 566T175 555T183 540T186 520Q186 498 172 481T127 463Z"></path><path id="MJMAIN-34" stroke-width="1" d="M462 0Q444 3 333 3Q217 3 199 0H190V46H221Q241 46 248 46T265 48T279 53T286 61Q287 63 287 115V165H28V211L179 442Q332 674 334 675Q336 677 355 677H373L379 671V211H471V165H379V114Q379 73 379 66T385 54Q393 47 442 46H471V0H462ZM293 211V545L74 212L183 211H293Z"></path><path id="MJMAIN-35" stroke-width="1" d="M164 157Q164 133 148 117T109 101H102Q148 22 224 22Q294 22 326 82Q345 115 345 210Q345 313 318 349Q292 382 260 382H254Q176 382 136 314Q132 307 129 306T114 304Q97 304 95 310Q93 314 93 485V614Q93 664 98 664Q100 666 102 666Q103 666 123 658T178 642T253 634Q324 634 389 662Q397 666 402 666Q410 666 410 648V635Q328 538 205 538Q174 538 149 544L139 546V374Q158 388 169 396T205 412T256 420Q337 420 393 355T449 201Q449 109 385 44T229 -22Q148 -22 99 32T50 154Q50 178 61 192T84 210T107 214Q132 214 148 197T164 157Z"></path><path id="MJMAIN-36" stroke-width="1" d="M42 313Q42 476 123 571T303 666Q372 666 402 630T432 550Q432 525 418 510T379 495Q356 495 341 509T326 548Q326 592 373 601Q351 623 311 626Q240 626 194 566Q147 500 147 364L148 360Q153 366 156 373Q197 433 263 433H267Q313 433 348 414Q372 400 396 374T435 317Q456 268 456 210V192Q456 169 451 149Q440 90 387 34T253 -22Q225 -22 199 -14T143 16T92 75T56 172T42 313ZM257 397Q227 397 205 380T171 335T154 278T148 216Q148 133 160 97T198 39Q222 21 251 21Q302 21 329 59Q342 77 347 104T352 209Q352 289 347 316T329 361Q302 397 257 397Z"></path><path id="MJMAIN-37" stroke-width="1" d="M55 458Q56 460 72 567L88 674Q88 676 108 676H128V672Q128 662 143 655T195 646T364 644H485V605L417 512Q408 500 387 472T360 435T339 403T319 367T305 330T292 284T284 230T278 162T275 80Q275 66 275 52T274 28V19Q270 2 255 -10T221 -22Q210 -22 200 -19T179 0T168 40Q168 198 265 368Q285 400 349 489L395 552H302Q128 552 119 546Q113 543 108 522T98 479L95 458V455H55V458Z"></path><path id="MJMAIN-2E" stroke-width="1" d="M78 60Q78 84 95 102T138 120Q162 120 180 104T199 61Q199 36 182 18T139 0T96 17T78 60Z"></path><path id="MJMATHI-6F" stroke-width="1" d="M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z"></path><path id="MJMAIN-79" stroke-width="1" d="M69 -66Q91 -66 104 -80T118 -116Q118 -134 109 -145T91 -160Q84 -163 97 -166Q104 -168 111 -168Q131 -168 148 -159T175 -138T197 -106T213 -75T225 -43L242 0L170 183Q150 233 125 297Q101 358 96 368T80 381Q79 382 78 382Q66 385 34 385H19V431H26L46 430Q65 430 88 429T122 428Q129 428 142 428T171 429T200 430T224 430L233 431H241V385H232Q183 385 185 366L286 112Q286 113 332 227L376 341V350Q376 365 366 373T348 383T334 385H331V431H337H344Q351 431 361 431T382 430T405 429T422 429Q477 429 503 431H508V385H497Q441 380 422 345Q420 343 378 235T289 9T227 -131Q180 -204 113 -204Q69 -204 44 -177T19 -116Q19 -89 35 -78T69 -66Z"></path><path id="MJMATHI-41" stroke-width="1" d="M208 74Q208 50 254 46Q272 46 272 35Q272 34 270 22Q267 8 264 4T251 0Q249 0 239 0T205 1T141 2Q70 2 50 0H42Q35 7 35 11Q37 38 48 46H62Q132 49 164 96Q170 102 345 401T523 704Q530 716 547 716H555H572Q578 707 578 706L606 383Q634 60 636 57Q641 46 701 46Q726 46 726 36Q726 34 723 22Q720 7 718 4T704 0Q701 0 690 0T651 1T578 2Q484 2 455 0H443Q437 6 437 9T439 27Q443 40 445 43L449 46H469Q523 49 533 63L521 213H283L249 155Q208 86 208 74ZM516 260Q516 271 504 416T490 562L463 519Q447 492 400 412L310 260L413 259Q516 259 516 260Z"></path><path id="MJMAIN-2032" stroke-width="1" d="M79 43Q73 43 52 49T30 61Q30 68 85 293T146 528Q161 560 198 560Q218 560 240 545T262 501Q262 496 260 486Q259 479 173 263T84 45T79 43Z"></path><path id="MJMAIN-38" stroke-width="1" d="M70 417T70 494T124 618T248 666Q319 666 374 624T429 515Q429 485 418 459T392 417T361 389T335 371T324 363L338 354Q352 344 366 334T382 323Q457 264 457 174Q457 95 399 37T249 -22Q159 -22 101 29T43 155Q43 263 172 335L154 348Q133 361 127 368Q70 417 70 494ZM286 386L292 390Q298 394 301 396T311 403T323 413T334 425T345 438T355 454T364 471T369 491T371 513Q371 556 342 586T275 624Q268 625 242 625Q201 625 165 599T128 534Q128 511 141 492T167 463T217 431Q224 426 228 424L286 386ZM250 21Q308 21 350 55T392 137Q392 154 387 169T375 194T353 216T330 234T301 253T274 270Q260 279 244 289T218 306L210 311Q204 311 181 294T133 239T107 157Q107 98 150 60T250 21Z"></path><path id="MJMAIN-39" stroke-width="1" d="M352 287Q304 211 232 211Q154 211 104 270T44 396Q42 412 42 436V444Q42 537 111 606Q171 666 243 666Q245 666 249 666T257 665H261Q273 665 286 663T323 651T370 619T413 560Q456 472 456 334Q456 194 396 97Q361 41 312 10T208 -22Q147 -22 108 7T68 93T121 149Q143 149 158 135T173 96Q173 78 164 65T148 49T135 44L131 43Q131 41 138 37T164 27T206 22H212Q272 22 313 86Q352 142 352 280V287ZM244 248Q292 248 321 297T351 430Q351 508 343 542Q341 552 337 562T323 588T293 615T246 625Q208 625 181 598Q160 576 154 546T147 441Q147 358 152 329T172 282Q197 248 244 248Z"></path><path id="MJMAIN-30" stroke-width="1" d="M96 585Q152 666 249 666Q297 666 345 640T423 548Q460 465 460 320Q460 165 417 83Q397 41 362 16T301 -15T250 -22Q224 -22 198 -16T137 16T82 83Q39 165 39 320Q39 494 96 585ZM321 597Q291 629 250 629Q208 629 178 597Q153 571 145 525T137 333Q137 175 145 125T181 46Q209 16 250 16Q290 16 318 46Q347 76 354 130T362 333Q362 478 354 524T321 597Z"></path><path id="MJMATHI-69" stroke-width="1" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369T98 420T158 442Q197 442 223 419T250 357Q250 340 236 301T196 196T154 83Q149 61 149 51Q149 26 166 26Q175 26 185 29T208 43T235 78T260 137Q263 149 265 151T282 153Q302 153 302 143Q302 135 293 112T268 61T223 11T161 -11Q129 -11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-3B4" stroke-width="1" d="M195 609Q195 656 227 686T302 717Q319 716 351 709T407 697T433 690Q451 682 451 662Q451 644 438 628T403 612Q382 612 348 641T288 671T249 657T235 628Q235 584 334 463Q401 379 401 292Q401 169 340 80T205 -10H198Q127 -10 83 36T36 153Q36 286 151 382Q191 413 252 434Q252 435 245 449T230 481T214 521T201 566T195 609ZM112 130Q112 83 136 55T204 27Q233 27 256 51T291 111T309 178T316 232Q316 267 309 298T295 344T269 400L259 396Q215 381 183 342T137 256T118 179T112 130Z"></path><path id="MJMATHI-6A" stroke-width="1" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405Q240 405 220 393T185 362T161 325T144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432Q341 418 354 401T367 348V332L318 133Q267 -67 264 -75Q246 -125 194 -164T75 -204Q25 -204 7 -183T-12 -137Q-12 -110 7 -91T53 -71Q70 -71 82 -81T95 -112Q95 -148 63 -167Q69 -168 77 -168Q111 -168 139 -140T182 -74L193 -32Q204 11 219 72T251 197T278 308T289 365Q289 372 288 376Z"></path><path id="MJMATHI-6E" stroke-width="1" d="M21 287Q22 293 24 303T36 341T56 388T89 425T135 442Q171 442 195 424T225 390T231 369Q231 367 232 367L243 378Q304 442 382 442Q436 442 469 415T503 336T465 179T427 52Q427 26 444 26Q450 26 453 27Q482 32 505 65T540 145Q542 153 560 153Q580 153 580 145Q580 144 576 130Q568 101 554 73T508 17T439 -10Q392 -10 371 17T350 73Q350 92 386 193T423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 180T152 343Q153 348 153 366Q153 405 129 405Q91 405 66 305Q60 285 60 284Q58 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-65" stroke-width="1" d="M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z"></path><path id="MJMAIN-D7" stroke-width="1" d="M630 29Q630 9 609 9Q604 9 587 25T493 118L389 222L284 117Q178 13 175 11Q171 9 168 9Q160 9 154 15T147 29Q147 36 161 51T255 146L359 250L255 354Q174 435 161 449T147 471Q147 480 153 485T168 490Q173 490 175 489Q178 487 284 383L389 278L493 382Q570 459 587 475T609 491Q630 491 630 471Q630 464 620 453T522 355L418 250L522 145Q606 61 618 48T630 29Z"></path><path id="MJMATHI-6D" stroke-width="1" d="M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMAIN-5B" stroke-width="1" d="M118 -250V750H255V710H158V-210H255V-250H118Z"></path><path id="MJMATHI-73" stroke-width="1" d="M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z"></path><path id="MJMAIN-5D" stroke-width="1" d="M22 710V750H159V-250H22V-210H119V710H22Z"></path><path id="MJSZ4-23A1" stroke-width="1" d="M319 -645V1154H666V1070H403V-645H319Z"></path><path id="MJSZ4-23A3" stroke-width="1" d="M319 -644V1155H403V-560H666V-644H319Z"></path><path id="MJSZ4-23A2" stroke-width="1" d="M319 0V602H403V0H319Z"></path><path id="MJSZ4-23A4" stroke-width="1" d="M0 1070V1154H347V-645H263V1070H0Z"></path><path id="MJSZ4-23A6" stroke-width="1" d="M263 -560V1155H347V-644H0V-560H263Z"></path><path id="MJSZ4-23A5" stroke-width="1" d="M263 0V602H347V0H263Z"></path><path id="MJMATHI-75" stroke-width="1" d="M21 287Q21 295 30 318T55 370T99 420T158 442Q204 442 227 417T250 358Q250 340 216 246T182 105Q182 62 196 45T238 27T291 44T328 78L339 95Q341 99 377 247Q407 367 413 387T427 416Q444 431 463 431Q480 431 488 421T496 402L420 84Q419 79 419 68Q419 43 426 35T447 26Q469 29 482 57T512 145Q514 153 532 153Q551 153 551 144Q550 139 549 130T540 98T523 55T498 17T462 -8Q454 -10 438 -10Q372 -10 347 46Q345 45 336 36T318 21T296 6T267 -6T233 -11Q189 -11 155 7Q103 38 103 113Q103 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-78" stroke-width="1" d="M52 289Q59 331 106 386T222 442Q257 442 286 424T329 379Q371 442 430 442Q467 442 494 420T522 361Q522 332 508 314T481 292T458 288Q439 288 427 299T415 328Q415 374 465 391Q454 404 425 404Q412 404 406 402Q368 386 350 336Q290 115 290 78Q290 50 306 38T341 26Q378 26 414 59T463 140Q466 150 469 151T485 153H489Q504 153 504 145Q504 144 502 134Q486 77 440 33T333 -11Q263 -11 227 52Q186 -10 133 -10H127Q78 -10 57 16T35 71Q35 103 54 123T99 143Q142 143 142 101Q142 81 130 66T107 46T94 41L91 40Q91 39 97 36T113 29T132 26Q168 26 194 71Q203 87 217 139T245 247T261 313Q266 340 266 352Q266 380 251 392T217 404Q177 404 142 372T93 290Q91 281 88 280T72 278H58Q52 284 52 289Z"></path><path id="MJMATHI-77" stroke-width="1" d="M580 385Q580 406 599 424T641 443Q659 443 674 425T690 368Q690 339 671 253Q656 197 644 161T609 80T554 12T482 -11Q438 -11 404 5T355 48Q354 47 352 44Q311 -11 252 -11Q226 -11 202 -5T155 14T118 53T104 116Q104 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Q21 293 29 315T52 366T96 418T161 441Q204 441 227 416T250 358Q250 340 217 250T184 111Q184 65 205 46T258 26Q301 26 334 87L339 96V119Q339 122 339 128T340 136T341 143T342 152T345 165T348 182T354 206T362 238T373 281Q402 395 406 404Q419 431 449 431Q468 431 475 421T483 402Q483 389 454 274T422 142Q420 131 420 107V100Q420 85 423 71T442 42T487 26Q558 26 600 148Q609 171 620 213T632 273Q632 306 619 325T593 357T580 385Z"></path><path id="MJMATHI-6C" stroke-width="1" d="M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z"></path><path id="MJMAIN-6E" stroke-width="1" d="M41 46H55Q94 46 102 60V68Q102 77 102 91T102 122T103 161T103 203Q103 234 103 269T102 328V351Q99 370 88 376T43 385H25V408Q25 431 27 431L37 432Q47 433 65 434T102 436Q119 437 138 438T167 441T178 442H181V402Q181 364 182 364T187 369T199 384T218 402T247 421T285 437Q305 442 336 442Q450 438 463 329Q464 322 464 190V104Q464 66 466 59T477 49Q498 46 526 46H542V0H534L510 1Q487 2 460 2T422 3Q319 3 310 0H302V46H318Q379 46 379 62Q380 64 380 200Q379 335 378 343Q372 371 358 385T334 402T308 404Q263 404 229 370Q202 343 195 315T187 232V168V108Q187 78 188 68T191 55T200 49Q221 46 249 46H265V0H257L234 1Q210 2 183 2T145 3Q42 3 33 0H25V46H41Z"></path><path id="MJMAIN-65" stroke-width="1" d="M28 218Q28 273 48 318T98 391T163 433T229 448Q282 448 320 430T378 380T406 316T415 245Q415 238 408 231H126V216Q126 68 226 36Q246 30 270 30Q312 30 342 62Q359 79 369 104L379 128Q382 131 395 131H398Q415 131 415 121Q415 117 412 108Q393 53 349 21T250 -11Q155 -11 92 58T28 218ZM333 275Q322 403 238 411H236Q228 411 220 410T195 402T166 381T143 340T127 274V267H333V275Z"></path><path id="MJMAIN-74" stroke-width="1" d="M27 422Q80 426 109 478T141 600V615H181V431H316V385H181V241Q182 116 182 100T189 68Q203 29 238 29Q282 29 292 100Q293 108 293 146V181H333V146V134Q333 57 291 17Q264 -10 221 -10Q187 -10 162 2T124 33T105 68T98 100Q97 107 97 248V385H18V422H27Z"></path><path id="MJMAIN-2202" stroke-width="1" d="M202 508Q179 508 169 520T158 547Q158 557 164 577T185 624T230 675T301 710L333 715H345Q378 715 384 714Q447 703 489 661T549 568T566 457Q566 362 519 240T402 53Q321 -22 223 -22Q123 -22 73 56Q42 102 42 148V159Q42 276 129 370T322 465Q383 465 414 434T455 367L458 378Q478 461 478 515Q478 603 437 639T344 676Q266 676 223 612Q264 606 264 572Q264 547 246 528T202 508ZM430 306Q430 372 401 400T333 428Q270 428 222 382Q197 354 183 323T150 221Q132 149 132 116Q132 21 232 21Q244 21 250 22Q327 35 374 112Q389 137 409 196T430 306Z"></path><path id="MJMAIN-61" stroke-width="1" d="M137 305T115 305T78 320T63 359Q63 394 97 421T218 448Q291 448 336 416T396 340Q401 326 401 309T402 194V124Q402 76 407 58T428 40Q443 40 448 56T453 109V145H493V106Q492 66 490 59Q481 29 455 12T400 -6T353 12T329 54V58L327 55Q325 52 322 49T314 40T302 29T287 17T269 6T247 -2T221 -8T190 -11Q130 -11 82 20T34 107Q34 128 41 147T68 188T116 225T194 253T304 268H318V290Q318 324 312 340Q290 411 215 411Q197 411 181 410T156 406T148 403Q170 388 170 359Q170 334 154 320ZM126 106Q126 75 150 51T209 26Q247 26 276 49T315 109Q317 116 318 175Q318 233 317 233Q309 233 296 232T251 223T193 203T147 166T126 106Z"></path><path id="MJMATHI-54" stroke-width="1" d="M40 437Q21 437 21 445Q21 450 37 501T71 602L88 651Q93 669 101 677H569H659Q691 677 697 676T704 667Q704 661 687 553T668 444Q668 437 649 437Q640 437 637 437T631 442L629 445Q629 451 635 490T641 551Q641 586 628 604T573 629Q568 630 515 631Q469 631 457 630T439 622Q438 621 368 343T298 60Q298 48 386 46Q418 46 427 45T436 36Q436 31 433 22Q429 4 424 1L422 0Q419 0 415 0Q410 0 363 1T228 2Q99 2 64 0H49Q43 6 43 9T45 27Q49 40 55 46H83H94Q174 46 189 55Q190 56 191 56Q196 59 201 76T241 233Q258 301 269 344Q339 619 339 625Q339 630 310 630H279Q212 630 191 624Q146 614 121 583T67 467Q60 445 57 441T43 437H40Z"></path><path id="MJMATHI-64" stroke-width="1" d="M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z"></path><path id="MJMATHI-61" stroke-width="1" d="M33 157Q33 258 109 349T280 441Q331 441 370 392Q386 422 416 422Q429 422 439 414T449 394Q449 381 412 234T374 68Q374 43 381 35T402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487Q506 153 506 144Q506 138 501 117T481 63T449 13Q436 0 417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157ZM351 328Q351 334 346 350T323 385T277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q217 26 254 59T298 110Q300 114 325 217T351 328Z"></path><path id="MJMATHI-6B" stroke-width="1" d="M121 647Q121 657 125 670T137 683Q138 683 209 688T282 694Q294 694 294 686Q294 679 244 477Q194 279 194 272Q213 282 223 291Q247 309 292 354T362 415Q402 442 438 442Q468 442 485 423T503 369Q503 344 496 327T477 302T456 291T438 288Q418 288 406 299T394 328Q394 353 410 369T442 390L458 393Q446 405 434 405H430Q398 402 367 380T294 316T228 255Q230 254 243 252T267 246T293 238T320 224T342 206T359 180T365 147Q365 130 360 106T354 66Q354 26 381 26Q429 26 459 145Q461 153 479 153H483Q499 153 499 144Q499 139 496 130Q455 -11 378 -11Q333 -11 305 15T277 90Q277 108 280 121T283 145Q283 167 269 183T234 206T200 217T182 220H180Q168 178 159 139T145 81T136 44T129 20T122 7T111 -2Q98 -11 83 -11Q66 -11 57 -1T48 16Q48 26 85 176T158 471L195 616Q196 629 188 632T149 637H144Q134 637 131 637T124 640T121 647Z"></path><path id="MJMATHI-45" stroke-width="1" d="M492 213Q472 213 472 226Q472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272Q690 274 698 274Q718 274 718 261Q613 7 608 2Q605 0 322 0H133Q31 0 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669Q764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528Q704 558 697 578T678 609T643 625T596 632T532 634H485Q397 633 392 631Q388 629 386 622Q385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492T572 491T574 487T577 483L544 351Q511 218 508 216Q505 213 492 213Z"></path><path id="MJSZ2-220F" stroke-width="1" d="M220 812Q220 813 218 819T214 829T208 840T199 853T185 866T166 878T140 887T107 893T66 896H56V950H1221V896H1211Q1080 896 1058 812V-311Q1076 -396 1211 -396H1221V-450H725V-396H735Q864 -396 888 -314Q889 -312 889 -311V896H388V292L389 -311Q405 -396 542 -396H552V-450H56V-396H66Q195 -396 219 -314Q220 -312 220 -311V812Z"></path><path id="MJMAIN-2207" stroke-width="1" d="M46 676Q46 679 51 683H781Q786 679 786 676Q786 674 617 326T444 -26Q439 -33 416 -33T388 -26Q385 -22 216 326T46 676ZM697 596Q697 597 445 597T193 596Q195 591 319 336T445 80L697 596Z"></path><path id="MJSZ2-2211" stroke-width="1" d="M60 948Q63 950 665 950H1267L1325 815Q1384 677 1388 669H1348L1341 683Q1320 724 1285 761Q1235 809 1174 838T1033 881T882 898T699 902H574H543H251L259 891Q722 258 724 252Q725 250 724 246Q721 243 460 -56L196 -356Q196 -357 407 -357Q459 -357 548 -357T676 -358Q812 -358 896 -353T1063 -332T1204 -283T1307 -196Q1328 -170 1348 -124H1388Q1388 -125 1381 -145T1356 -210T1325 -294L1267 -449L666 -450Q64 -450 61 -448Q55 -446 55 -439Q55 -437 57 -433L590 177Q590 178 557 222T452 366T322 544L56 909L55 924Q55 945 60 948Z"></path><path id="MJMATHI-76" stroke-width="1" d="M173 380Q173 405 154 405Q130 405 104 376T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Q21 294 29 316T53 368T97 419T160 441Q202 441 225 417T249 361Q249 344 246 335Q246 329 231 291T200 202T182 113Q182 86 187 69Q200 26 250 26Q287 26 319 60T369 139T398 222T409 277Q409 300 401 317T383 343T365 361T357 383Q357 405 376 424T417 443Q436 443 451 425T467 367Q467 340 455 284T418 159T347 40T241 -11Q177 -11 139 22Q102 54 102 117Q102 148 110 181T151 298Q173 362 173 380Z"></path><path id="MJSZ1-5B" stroke-width="1" d="M202 -349V850H394V810H242V-309H394V-349H202Z"></path><path id="MJSZ1-5D" stroke-width="1" d="M22 810V850H214V-349H22V-309H174V810H22Z"></path><path id="MJMAIN-2225" stroke-width="1" d="M133 736Q138 750 153 750Q164 750 170 739Q172 735 172 250T170 -239Q164 -250 152 -250Q144 -250 138 -244L137 -243Q133 -241 133 -179T132 250Q132 731 133 736ZM329 739Q334 750 346 750Q353 750 361 744L362 743Q366 741 366 679T367 250T367 -178T362 -243L361 -244Q355 -250 347 -250Q335 -250 329 -239Q327 -235 327 250T329 739Z"></path><path id="MJAMS-2A7D" stroke-width="1" d="M674 636Q682 636 688 630T694 615T687 601Q686 600 417 472L151 346L399 228Q687 92 691 87Q694 81 694 76Q694 58 676 56H670L382 192Q92 329 90 331Q83 336 83 348Q84 359 96 365Q104 369 382 500T665 634Q669 636 674 636ZM94 170Q102 172 104 172Q110 171 254 103T535 -30T678 -98Q694 -106 694 -118Q694 -136 676 -138H670L382 -2Q92 135 90 137Q83 142 83 154Q84 164 94 170Z"></path><path id="MJMATHI-3B2" stroke-width="1" d="M29 -194Q23 -188 23 -186Q23 -183 102 134T186 465Q208 533 243 584T309 658Q365 705 429 705H431Q493 705 533 667T573 570Q573 465 469 396L482 383Q533 332 533 252Q533 139 448 65T257 -10Q227 -10 203 -2T165 17T143 40T131 59T126 65L62 -188Q60 -194 42 -194H29ZM353 431Q392 431 427 419L432 422Q436 426 439 429T449 439T461 453T472 471T484 495T493 524T501 560Q503 569 503 593Q503 611 502 616Q487 667 426 667Q384 667 347 643T286 582T247 514T224 455Q219 439 186 308T152 168Q151 163 151 147Q151 99 173 68Q204 26 260 26Q302 26 349 51T425 137Q441 171 449 214T457 279Q457 337 422 372Q380 358 347 358H337Q258 358 258 389Q258 396 261 403Q275 431 353 431Z"></path><path id="MJMAIN-7B" stroke-width="1" d="M434 -231Q434 -244 428 -250H410Q281 -250 230 -184Q225 -177 222 -172T217 -161T213 -148T211 -133T210 -111T209 -84T209 -47T209 0Q209 21 209 53Q208 142 204 153Q203 154 203 155Q189 191 153 211T82 231Q71 231 68 234T65 250T68 266T82 269Q116 269 152 289T203 345Q208 356 208 377T209 529V579Q209 634 215 656T244 698Q270 724 324 740Q361 748 377 749Q379 749 390 749T408 750H428Q434 744 434 732Q434 719 431 716Q429 713 415 713Q362 710 332 689T296 647Q291 634 291 499V417Q291 370 288 353T271 314Q240 271 184 255L170 250L184 245Q202 239 220 230T262 196T290 137Q291 131 291 1Q291 -134 296 -147Q306 -174 339 -192T415 -213Q429 -213 431 -216Q434 -219 434 -231Z"></path><path id="MJMAIN-2260" stroke-width="1" d="M166 -215T159 -215T147 -212T141 -204T139 -197Q139 -190 144 -183L306 133H70Q56 140 56 153Q56 168 72 173H327L406 327H72Q56 332 56 347Q56 360 70 367H426Q597 702 602 707Q605 716 618 716Q625 716 630 712T636 703T638 696Q638 692 471 367H707Q722 359 722 347Q722 336 708 328L451 327L371 173H708Q722 163 722 153Q722 140 707 133H351Q175 -210 170 -212Q166 -215 159 -215Z"></path><path id="MJSZ3-7B" stroke-width="1" d="M618 -943L612 -949H582L568 -943Q472 -903 411 -841T332 -703Q327 -682 327 -653T325 -350Q324 -28 323 -18Q317 24 301 61T264 124T221 171T179 205T147 225T132 234Q130 238 130 250Q130 255 130 258T131 264T132 267T134 269T139 272T144 275Q207 308 256 367Q310 436 323 519Q324 529 325 851Q326 1124 326 1154T332 1205Q369 1358 566 1443L582 1450H612L618 1444V1429Q618 1413 616 1411L608 1406Q599 1402 585 1393T552 1372T515 1343T479 1305T449 1257T429 1200Q425 1180 425 1152T423 851Q422 579 422 549T416 498Q407 459 388 424T346 364T297 318T250 284T214 264T197 254L188 251L205 242Q290 200 345 138T416 3Q421 -18 421 -48T423 -349Q423 -397 423 -472Q424 -677 428 -694Q429 -697 429 -699Q434 -722 443 -743T465 -782T491 -816T519 -845T548 -868T574 -886T595 -899T610 -908L616 -910Q618 -912 618 -928V-943Z"></path><path id="MJMATHI-7A" stroke-width="1" d="M347 338Q337 338 294 349T231 360Q211 360 197 356T174 346T162 335T155 324L153 320Q150 317 138 317Q117 317 117 325Q117 330 120 339Q133 378 163 406T229 440Q241 442 246 442Q271 442 291 425T329 392T367 375Q389 375 411 408T434 441Q435 442 449 442H462Q468 436 468 434Q468 430 463 420T449 399T432 377T418 358L411 349Q368 298 275 214T160 106L148 94L163 93Q185 93 227 82T290 71Q328 71 360 90T402 140Q406 149 409 151T424 153Q443 153 443 143Q443 138 442 134Q425 72 376 31T278 -11Q252 -11 232 6T193 40T155 57Q111 57 76 -3Q70 -11 59 -11H54H41Q35 -5 35 -2Q35 13 93 84Q132 129 225 214T340 322Q352 338 347 338Z"></path><path id="MJSZ1-2211" stroke-width="1" d="M61 748Q64 750 489 750H913L954 640Q965 609 976 579T993 533T999 516H979L959 517Q936 579 886 621T777 682Q724 700 655 705T436 710H319Q183 710 183 709Q186 706 348 484T511 259Q517 250 513 244L490 216Q466 188 420 134T330 27L149 -187Q149 -188 362 -188Q388 -188 436 -188T506 -189Q679 -189 778 -162T936 -43Q946 -27 959 6H999L913 -249L489 -250Q65 -250 62 -248Q56 -246 56 -239Q56 -234 118 -161Q186 -81 245 -11L428 206Q428 207 242 462L57 717L56 728Q56 744 61 748Z"></path><path id="MJMATHI-79" stroke-width="1" d="M21 287Q21 301 36 335T84 406T158 442Q199 442 224 419T250 355Q248 336 247 334Q247 331 231 288T198 191T182 105Q182 62 196 45T238 27Q261 27 281 38T312 61T339 94Q339 95 344 114T358 173T377 247Q415 397 419 404Q432 431 462 431Q475 431 483 424T494 412T496 403Q496 390 447 193T391 -23Q363 -106 294 -155T156 -205Q111 -205 77 -183T43 -117Q43 -95 50 -80T69 -58T89 -48T106 -45Q150 -45 150 -87Q150 -107 138 -122T115 -142T102 -147L99 -148Q101 -153 118 -160T152 -167H160Q177 -167 186 -165Q219 -156 247 -127T290 -65T313 -9T321 21L315 17Q309 13 296 6T270 -6Q250 -11 231 -11Q185 -11 150 11T104 82Q103 89 103 113Q103 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-4C" stroke-width="1" d="M228 637Q194 637 192 641Q191 643 191 649Q191 673 202 682Q204 683 217 683Q271 680 344 680Q485 680 506 683H518Q524 677 524 674T522 656Q517 641 513 637H475Q406 636 394 628Q387 624 380 600T313 336Q297 271 279 198T252 88L243 52Q243 48 252 48T311 46H328Q360 46 379 47T428 54T478 72T522 106T564 161Q580 191 594 228T611 270Q616 273 628 273H641Q647 264 647 262T627 203T583 83T557 9Q555 4 553 3T537 0T494 -1Q483 -1 418 -1T294 0H116Q32 0 32 10Q32 17 34 24Q39 43 44 45Q48 46 59 46H65Q92 46 125 49Q139 52 144 61Q147 65 216 339T285 628Q285 635 228 637Z"></path><path id="MJMAIN-2C" stroke-width="1" d="M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z"></path><path id="MJMATHI-4E" stroke-width="1" d="M234 637Q231 637 226 637Q201 637 196 638T191 649Q191 676 202 682Q204 683 299 683Q376 683 387 683T401 677Q612 181 616 168L670 381Q723 592 723 606Q723 633 659 637Q635 637 635 648Q635 650 637 660Q641 676 643 679T653 683Q656 683 684 682T767 680Q817 680 843 681T873 682Q888 682 888 672Q888 650 880 642Q878 637 858 637Q787 633 769 597L620 7Q618 0 599 0Q585 0 582 2Q579 5 453 305L326 604L261 344Q196 88 196 79Q201 46 268 46H278Q284 41 284 38T282 19Q278 6 272 0H259Q228 2 151 2Q123 2 100 2T63 2T46 1Q31 1 31 10Q31 14 34 26T39 40Q41 46 62 46Q130 49 150 85Q154 91 221 362L289 634Q287 635 234 637Z"></path><path id="MJMAIN-2208" stroke-width="1" d="M84 250Q84 372 166 450T360 539Q361 539 377 539T419 540T469 540H568Q583 532 583 520Q583 511 570 501L466 500Q355 499 329 494Q280 482 242 458T183 409T147 354T129 306T124 272V270H568Q583 262 583 250T568 230H124V228Q124 207 134 177T167 112T231 48T328 7Q355 1 466 0H570Q583 -10 583 -20Q583 -32 568 -40H471Q464 -40 446 -40T417 -41Q262 -41 172 45Q84 127 84 250Z"></path><path id="MJMAIN-2217" stroke-width="1" d="M229 286Q216 420 216 436Q216 454 240 464Q241 464 245 464T251 465Q263 464 273 456T283 436Q283 419 277 356T270 286L328 328Q384 369 389 372T399 375Q412 375 423 365T435 338Q435 325 425 315Q420 312 357 282T289 250L355 219L425 184Q434 175 434 161Q434 146 425 136T401 125Q393 125 383 131T328 171L270 213Q283 79 283 63Q283 53 276 44T250 35Q231 35 224 44T216 63Q216 80 222 143T229 213L171 171Q115 130 110 127Q106 124 100 124Q87 124 76 134T64 161Q64 166 64 169T67 175T72 181T81 188T94 195T113 204T138 215T170 230T210 250L74 315Q65 324 65 338Q65 353 74 363T98 374Q106 374 116 368T171 328L229 286Z"></path></defs></svg></div><div id="MathJax_Message" style="display: none;"></div><div class="pace  pace-inactive"><div class="pace-progress" data-progress-text="100%" data-progress="99" style="width: 100%;">
  <div class="pace-progress-inner"></div>
</div>
<div class="pace-activity"></div></div>

    <div id="global-prompt-alert" class="hide alert alert-warning">
        <span id="global-prompt-message"></span>
        <a id="close-global-prompt-alert" href="https://zybuluo.com/hanbingtao/note/541458">[关闭]</a>
    </div>

    <!-- zybuluo's body -->
    







<!-- mdeditor's body -->






<div id="editor-reader-full" class="editor-reader-full-shown" style="position: static; padding-right: 75px;">
    <div id="reader-full-topInfo" class="reader-full-topInfo-shown">
        <span>
            <code>@hanbingtao</code>
        </span>
        <code><span class="article-updated-date">2017-08-28 19:54</span></code>
        <code><span>字数 </span><span class="article-characters">23656</span></code>
        <code><span>阅读 </span><span class="article-read">193836</span></code>
    </div>
    <div id="wmd-preview" class="wmd-preview wmd-preview-full-reader" data-medium-element="true"><div class="md-section-divider"></div><div class="md-section-divider"></div><h1 data-anchor-id="yz6y" id="零基础入门深度学习5-循环神经网络">零基础入门深度学习(5) - 循环神经网络</h1><p data-anchor-id="07q4"><code>机器学习</code> <code>深度学习入门</code></p><hr><p data-anchor-id="id7r"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-06627c71f0d8c0dc.jpg" alt=""></p><blockquote data-anchor-id="gcom" class="white-blockquote">
  <p>无论即将到来的是大数据时代还是人工智能时代，亦或是传统行业使用人工智能在云上处理大数据的时代，作为一个有理想有追求的程序员，不懂深度学习（Deep Learning）这个超热的技术，会不会感觉马上就out了？现在救命稻草来了，《零基础入门深度学习》系列文章旨在讲帮助爱编程的你从零基础达到入门级水平。零基础意味着你不需要太多的数学知识，只要会写程序就行了，没错，这是专门为程序员写的文章。虽然文中会有很多公式你也许看不懂，但同时也会有更多的代码，程序员的你一定能看懂的（我周围是一群狂热的Clean Code程序员，所以我写的代码也不会很差）。</p>
</blockquote><div class="md-section-divider"></div><h2 data-anchor-id="hcb8" id="文章列表">文章列表</h2><p data-anchor-id="lnzm"><a href="https://www.zybuluo.com/hanbingtao/note/433855" target="_blank">零基础入门深度学习(1) - 感知器</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/448086" target="_blank">零基础入门深度学习(2) - 线性单元和梯度下降</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/476663" target="_blank">零基础入门深度学习(3) - 神经网络和反向传播算法</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/485480" target="_blank">零基础入门深度学习(4) - 卷积神经网络</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/541458" target="_blank">零基础入门深度学习(5) - 循环神经网络</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/581764" target="_blank">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/626300" target="_blank">零基础入门深度学习(7) - 递归神经网络</a></p><div class="md-section-divider"></div><h2 data-anchor-id="b7lc" id="往期回顾">往期回顾</h2><p data-anchor-id="rh2n">在前面的文章系列文章中，我们介绍了全连接神经网络和卷积神经网络，以及它们的训练和使用。他们都只能单独的取处理一个个的输入，前一个输入和后一个输入是完全没有关系的。但是，某些任务需要能够更好的处理<strong>序列</strong>的信息，即前面的输入和后面的输入是有关系的。比如，当我们在理解一句话意思时，孤立的理解这句话的每个词是不够的，我们需要处理这些词连接起来的整个<strong>序列</strong>；当我们处理视频的时候，我们也不能只单独的去分析每一帧，而要分析这些帧连接起来的整个<strong>序列</strong>。这时，就需要用到深度学习领域中另一类非常重要神经网络：<strong>循环神经网络(Recurrent Neural Network)</strong>。RNN种类很多，也比较绕脑子。不过读者不用担心，本文将一如既往的对复杂的东西剥茧抽丝，帮助您理解RNNs以及它的训练算法，并动手实现一个<strong>循环神经网络</strong>。</p><div class="md-section-divider"></div><h2 data-anchor-id="bifn" id="语言模型">语言模型</h2><p data-anchor-id="ah0a">RNN是在<strong>自然语言处理</strong>领域中最先被用起来的，比如，RNN可以为<strong>语言模型</strong>来建模。那么，什么是语言模型呢？</p><p data-anchor-id="pdef">我们可以和电脑玩一个游戏，我们写出一个句子前面的一些词，然后，让电脑帮我们写下接下来的一个词。比如下面这句：</p><blockquote data-anchor-id="2b84" class="white-blockquote">
  <p>我昨天上学迟到了，老师批评了____。</p>
</blockquote><p data-anchor-id="dtmk">我们给电脑展示了这句话前面这些词，然后，让电脑写下接下来的一个词。在这个例子中，接下来的这个词最有可能是『我』，而不太可能是『小明』，甚至是『吃饭』。</p><p data-anchor-id="2pls"><strong>语言模型</strong>就是这样的东西：给定一个一句话前面的部分，预测接下来最有可能的一个词是什么。</p><p data-anchor-id="9ui4"><strong>语言模型</strong>是对一种语言的特征进行建模，它有很多很多用处。比如在语音转文本(STT)的应用中，声学模型输出的结果，往往是若干个可能的候选词，这时候就需要<strong>语言模型</strong>来从这些候选词中选择一个最可能的。当然，它同样也可以用在图像到文本的识别中(OCR)。</p><p data-anchor-id="v19n">使用RNN之前，语言模型主要是采用N-Gram。N可以是一个自然数，比如2或者3。它的含义是，假设一个词出现的概率只与前面N个词相关。我们以2-Gram为例。首先，对前面的一句话进行切词：</p><blockquote data-anchor-id="c0ja" class="white-blockquote">
  <p>我 昨天 上学 迟到 了 ，老师 批评 了 ____。</p>
</blockquote><p data-anchor-id="aqcz">如果用2-Gram进行建模，那么电脑在预测的时候，只会看到前面的『了』，然后，电脑会在语料库中，搜索『了』后面最可能的一个词。不管最后电脑选的是不是『我』，我们都知道这个模型是不靠谱的，因为『了』前面说了那么一大堆实际上是没有用到的。如果是3-Gram模型呢，会搜索『批评了』后面最可能的词，感觉上比2-Gram靠谱了不少，但还是远远不够的。因为这句话最关键的信息『我』，远在9个词之前！</p><p data-anchor-id="5c2c">现在读者可能会想，可以提升继续提升N的值呀，比如4-Gram、5-Gram.......。实际上，这个想法是没有实用性的。因为我们想处理任意长度的句子，N设为多少都不合适；另外，模型的大小和N的关系是指数级的，4-Gram模型就会占用海量的存储空间。</p><p data-anchor-id="aoty">所以，该轮到RNN出场了，RNN理论上可以往前看(往后看)任意多个词。</p><div class="md-section-divider"></div><h2 data-anchor-id="kvwy" id="循环神经网络是啥">循环神经网络是啥</h2><p data-anchor-id="2qvc">循环神经网络种类繁多，我们先从最简单的基本循环神经网络开始吧。</p><div class="md-section-divider"></div><h3 data-anchor-id="gkeg" id="基本循环神经网络">基本循环神经网络</h3><p data-anchor-id="n5zu">下图是一个简单的循环神经网络如，它由输入层、一个隐藏层和一个输出层组成：</p><p data-anchor-id="ovr4"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-479f2a7488b91671.jpg" alt="" title=""></p><p data-anchor-id="9p4d">纳尼？！相信第一次看到这个玩意的读者内心和我一样是崩溃的。因为<strong>循环神经网络</strong>实在是太难画出来了，网上所有大神们都不得不用了这种抽象艺术手法。不过，静下心来仔细看看的话，其实也是很好理解的。如果把上面有W的那个带箭头的圈去掉，它就变成了最普通的<strong>全连接神经网络</strong>。x是一个向量，它表示<strong>输入层</strong>的值（这里面没有画出来表示神经元节点的圆圈）；s是一个向量，它表示<strong>隐藏层</strong>的值（这里隐藏层面画了一个节点，你也可以想象这一层其实是多个节点，节点数与向量s的维度相同）；U是输入层到隐藏层的<strong>权重矩阵</strong>（读者可以回到第三篇文章<a href="https://www.zybuluo.com/hanbingtao/note/476663" target="_blank">零基础入门深度学习(3) - 神经网络和反向传播算法</a>，看看我们是怎样用矩阵来表示<strong>全连接神经网络</strong>的计算的）；o也是一个向量，它表示<strong>输出层</strong>的值；V是隐藏层到输出层的<strong>权重矩阵</strong>。那么，现在我们来看看W是什么。<strong>循环神经网络</strong>的<strong>隐藏层</strong>的值s不仅仅取决于当前这次的输入x，还取决于上一次<strong>隐藏层</strong>的值s。<strong>权重矩阵</strong> W就是<strong>隐藏层</strong>上一次的值作为这一次的输入的权重。</p><p data-anchor-id="jlbj">如果我们把上面的图展开，<strong>循环神经网络</strong>也可以画成下面这个样子：</p><p data-anchor-id="vdof"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-cf18bb1f06e750a4.jpg" alt="" title=""></p><p data-anchor-id="7oe3">现在看上去就比较清楚了，这个网络在t时刻接收到输入<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-1-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 884.1191013989369 628.8854201102238" style="width: 2.056ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-1">\mathrm{x}_{t}</script>之后，隐藏层的值是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-2-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 750.1191013989369 645.8854201102238" style="width: 1.694ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-2">\mathrm{s}_{t}</script>，输出值是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-3-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 856.1191013989369 645.8854201102238" style="width: 1.935ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-3">\mathrm{o}_{t}</script>。关键一点是，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-4-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 750.1191013989369 645.8854201102238" style="width: 1.694ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-4">\mathrm{s}_t</script>的值不仅仅取决于<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-5-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 884.1191013989369 628.8854201102238" style="width: 2.056ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-5">\mathrm{x}_t</script>，还取决于<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-6-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 1654.5086745365313 645.8854201102238" style="width: 3.871ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-6">\mathrm{s}_{t-1}</script>。我们可以用下面的公式来表示<strong>循环神经网络</strong>的计算方法：</p><div class="md-section-divider"></div><p data-anchor-id="jum1"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-7-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1669.1955532389602 44270.673387096766 2838.3911064779204" style="width: 102.823ex; height: 6.532ex; vertical-align: -2.782ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(15704,0)"><g transform="translate(-15,0)"><g transform="translate(0,764)"><use href="#MJMAIN-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-213"></use></g><g transform="translate(106,-786)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g></g><g transform="translate(1119,0)"><g transform="translate(0,764)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-67" x="1056" y="0"></use><use href="#MJMAIN-28" x="1536" y="0"></use><use href="#MJMATHI-56" x="1926" y="0"></use><g transform="translate(2695,0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g><use href="#MJMAIN-29" x="3445" y="0"></use><use href="#MJMAIN-28" x="8835" y="0"></use><g transform="translate(9224,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-31" x="10058" y="0"></use><use href="#MJMAIN-29" x="10558" y="0"></use></g><g transform="translate(0,-786)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><use href="#MJMATHI-55" x="1996" y="0"></use><g transform="translate(2763,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="3870" y="0"></use><use href="#MJMATHI-57" x="4870" y="0"></use><g transform="translate(5919,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="7573" y="0"></use><use href="#MJMAIN-28" x="9963" y="0"></use><g transform="translate(10352,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-32" x="11186" y="0"></use><use href="#MJMAIN-29" x="11686" y="0"></use></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,764)"><g id="mjx-eqn-1"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-786)"><g id="mjx-eqn-2"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-7">
\begin{align}
\mathrm{o}_t&=g(V\mathrm{s}_t)\qquad\qquad\quad(式1)\\
\mathrm{s}_t&=f(U\mathrm{x}_t+W\mathrm{s}_{t-1})\qquad(式2)\\
\end{align}
</script><p></p><p data-anchor-id="bltz"><strong>式1</strong>是<strong>输出层</strong>的计算公式，输出层是一个<strong>全连接层</strong>，也就是它的每个节点都和隐藏层的每个节点相连。V是输出层的<strong>权重矩阵</strong>，g是<strong>激活函数</strong>。式2是隐藏层的计算公式，它是<strong>循环层</strong>。U是输入x的权重矩阵，W是上一次的值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-8-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 1654.5086745365313 645.8854201102238" style="width: 3.871ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-8">\mathrm{s}_{t-1}</script>作为这一次的输入的<strong>权重矩阵</strong>，f是<strong>激活函数</strong>。</p><p data-anchor-id="4kpk">从上面的公式我们可以看出，<strong>循环层</strong>和<strong>全连接层</strong>的区别就是<strong>循环层</strong>多了一个<strong>权重矩阵</strong> W。</p><p data-anchor-id="db9y">如果反复把<strong>式2</strong>带入到<strong>式1</strong>，我们将得到：</p><div class="md-section-divider"></div><p data-anchor-id="78h0"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-9-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3398.7000693679925 44270.673387096766 6297.400138735985" style="width: 102.823ex; height: 14.637ex; vertical-align: -6.774ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(9046,0)"><g transform="translate(-15,0)"><g transform="translate(0,2579)"><use href="#MJMAIN-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-213"></use></g></g><g transform="translate(1119,0)"><g transform="translate(0,2579)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-67" x="1056" y="0"></use><use href="#MJMAIN-28" x="1536" y="0"></use><use href="#MJMATHI-56" x="1926" y="0"></use><g transform="translate(2695,0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g><use href="#MJMAIN-29" x="3445" y="0"></use></g><g transform="translate(0,1277)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-56" x="1056" y="0"></use><use href="#MJMATHI-66" x="1825" y="0"></use><use href="#MJMAIN-28" x="2376" y="0"></use><use href="#MJMATHI-55" x="2765" y="0"></use><g transform="translate(3533,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="4639" y="0"></use><use href="#MJMATHI-57" x="5640" y="0"></use><g transform="translate(6688,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="8343" y="0"></use></g><g transform="translate(0,-25)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-56" x="1056" y="0"></use><use href="#MJMATHI-66" x="1825" y="0"></use><use href="#MJMAIN-28" x="2376" y="0"></use><use href="#MJMATHI-55" x="2765" y="0"></use><g transform="translate(3533,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="4639" y="0"></use><use href="#MJMATHI-57" x="5640" y="0"></use><use href="#MJMATHI-66" x="6688" y="0"></use><use href="#MJMAIN-28" x="7239" y="0"></use><use href="#MJMATHI-55" x="7628" y="0"></use><g transform="translate(8396,0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-2B" x="10407" y="0"></use><use href="#MJMATHI-57" x="11407" y="0"></use><g transform="translate(12456,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="14110" y="0"></use><use href="#MJMAIN-29" x="14500" y="0"></use></g><g transform="translate(0,-1327)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-56" x="1056" y="0"></use><use href="#MJMATHI-66" x="1825" y="0"></use><use href="#MJMAIN-28" x="2376" y="0"></use><use href="#MJMATHI-55" x="2765" y="0"></use><g transform="translate(3533,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="4639" y="0"></use><use href="#MJMATHI-57" x="5640" y="0"></use><use href="#MJMATHI-66" x="6688" y="0"></use><use href="#MJMAIN-28" x="7239" y="0"></use><use href="#MJMATHI-55" x="7628" y="0"></use><g transform="translate(8396,0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-2B" x="10407" y="0"></use><use href="#MJMATHI-57" x="11407" y="0"></use><use href="#MJMATHI-66" x="12456" y="0"></use><use href="#MJMAIN-28" x="13006" y="0"></use><use href="#MJMATHI-55" x="13396" y="0"></use><g transform="translate(14163,0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1140" y="0"></use></g></g><use href="#MJMAIN-2B" x="16174" y="0"></use><use href="#MJMATHI-57" x="17175" y="0"></use><g transform="translate(18223,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="19878" y="0"></use><use href="#MJMAIN-29" x="20267" y="0"></use><use href="#MJMAIN-29" x="20657" y="0"></use></g><g transform="translate(0,-2629)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-56" x="1056" y="0"></use><use href="#MJMATHI-66" x="1825" y="0"></use><use href="#MJMAIN-28" x="2376" y="0"></use><use href="#MJMATHI-55" x="2765" y="0"></use><g transform="translate(3533,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="4639" y="0"></use><use href="#MJMATHI-57" x="5640" y="0"></use><use href="#MJMATHI-66" x="6688" y="0"></use><use href="#MJMAIN-28" x="7239" y="0"></use><use href="#MJMATHI-55" x="7628" y="0"></use><g transform="translate(8396,0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-2B" x="10407" y="0"></use><use href="#MJMATHI-57" x="11407" y="0"></use><use href="#MJMATHI-66" x="12456" y="0"></use><use href="#MJMAIN-28" x="13006" y="0"></use><use href="#MJMATHI-55" x="13396" y="0"></use><g transform="translate(14163,0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1140" y="0"></use></g></g><use href="#MJMAIN-2B" x="16174" y="0"></use><use href="#MJMATHI-57" x="17175" y="0"></use><use href="#MJMATHI-66" x="18223" y="0"></use><use href="#MJMAIN-28" x="18774" y="0"></use><use href="#MJMATHI-55" x="19163" y="0"></use><g transform="translate(19931,0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="1140" y="0"></use></g></g><use href="#MJMAIN-2B" x="21719" y="0"></use><use href="#MJMAIN-2E" x="22498" y="0"></use><use href="#MJMAIN-2E" x="22943" y="0"></use><use href="#MJMAIN-2E" x="23388" y="0"></use><use href="#MJMAIN-29" x="23833" y="0"></use><use href="#MJMAIN-29" x="24223" y="0"></use><use href="#MJMAIN-29" x="24612" y="0"></use><use href="#MJMAIN-29" x="25002" y="0"></use></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,2579)"><g id="mjx-eqn-3"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,1277)"><g id="mjx-eqn-4"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-25)"><g id="mjx-eqn-5"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-1327)"><g id="mjx-eqn-6"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-2629)"><g id="mjx-eqn-7"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-9">
\begin{align}
\mathrm{o}_t&=g(V\mathrm{s}_t)\\
&=Vf(U\mathrm{x}_t+W\mathrm{s}_{t-1})\\
&=Vf(U\mathrm{x}_t+Wf(U\mathrm{x}_{t-1}+W\mathrm{s}_{t-2}))\\
&=Vf(U\mathrm{x}_t+Wf(U\mathrm{x}_{t-1}+Wf(U\mathrm{x}_{t-2}+W\mathrm{s}_{t-3})))\\
&=Vf(U\mathrm{x}_t+Wf(U\mathrm{x}_{t-1}+Wf(U\mathrm{x}_{t-2}+Wf(U\mathrm{x}_{t-3}+...))))
\end{align}
</script><p></p><p data-anchor-id="91fm">从上面可以看出，<strong>循环神经网络</strong>的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-10-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 841.1191013989369 638.8854201102238" style="width: 1.935ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="686" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-10">o_t</script>，是受前面历次输入值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-11-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 884.1191013989369 628.8854201102238" style="width: 2.056ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-11">\mathrm{x}_t</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-12-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 1788.5086745365313 628.8854201102238" style="width: 4.113ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-12">\mathrm{x}_{t-1}</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-13-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 1788.5086745365313 628.8854201102238" style="width: 4.113ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-13">\mathrm{x}_{t-2}</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-14-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 1788.5086745365313 636.6635947032757" style="width: 4.113ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-78"></use><g transform="translate(528,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-14">\mathrm{x}_{t-3}</script>、...影响的，这就是为什么<strong>循环神经网络</strong>可以往前看任意多个<strong>输入值</strong>的原因。</p><div class="md-section-divider"></div><h3 data-anchor-id="2e4v" id="双向循环神经网络">双向循环神经网络</h3><p data-anchor-id="l5b4">对于<strong>语言模型</strong>来说，很多时候光看前面的词是不够的，比如下面这句话：</p><blockquote data-anchor-id="jssd" class="white-blockquote">
  <p>我的手机坏了，我打算____一部新手机。</p>
</blockquote><p data-anchor-id="qp28">可以想象，如果我们只看横线前面的词，手机坏了，那么我是打算修一修？换一部新的？还是大哭一场？这些都是无法确定的。但如果我们也看到了横线后面的词是『一部新手机』，那么，横线上的词填『买』的概率就大得多了。</p><p data-anchor-id="4xzv">在上一小节中的<strong>基本循环神经网络</strong>是无法对此进行建模的，因此，我们需要<strong>双向循环神经网络</strong>，如下图所示：</p><p data-anchor-id="fgt4"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-039a45251aa5d220.png" alt="" title=""></p><p data-anchor-id="c2wn">当遇到这种从未来穿越回来的场景时，难免处于懵逼的状态。不过我们还是可以用屡试不爽的老办法：先分析一个特殊场景，然后再总结一般规律。我们先考虑上图中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-15-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 982.406943983867 711.462584576499" style="width: 2.298ex; height: 1.694ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="747" y="-340"></use></g></svg></span><script type="math/tex" id="MathJax-Element-15">\mathrm{y}_2</script>的计算。</p><p data-anchor-id="7pn8">从上图可以看出，<strong>双向卷积神经网络</strong>的隐藏层要保存两个值，一个A参与正向计算，另一个值A'参与反向计算。最终的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-16-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 982.406943983867 711.462584576499" style="width: 2.298ex; height: 1.694ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="747" y="-340"></use></g></svg></span><script type="math/tex" id="MathJax-Element-16">\mathrm{y}_2</script>取决于<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-17-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -736.2000693679926 1204.406943983867 906.1072455171717" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1061" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-17">A_2</script>和<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-18-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -736.2000693679926 1204.406943983867 1063.304268568599" style="width: 2.782ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1061" y="445"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1061" y="-435"></use></g></svg></span><script type="math/tex" id="MathJax-Element-18">A_2'</script>。其计算方法为：</p><div class="md-section-divider"></div><p data-anchor-id="hjzw"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-19-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -828.7789736136457 9097.378750168496 1155.883172814252" style="width: 21.169ex; height: 2.661ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="747" y="-340"></use><use href="#MJMAIN-3D" x="1260" y="0"></use><use href="#MJMATHI-67" x="2316" y="0"></use><use href="#MJMAIN-28" x="2796" y="0"></use><use href="#MJMATHI-56" x="3186" y="0"></use><g transform="translate(3955,0)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1061" y="-213"></use></g><use href="#MJMAIN-2B" x="5382" y="0"></use><g transform="translate(6383,0)"><use href="#MJMATHI-56"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1167" y="583"></use></g><g transform="translate(7503,0)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1061" y="445"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1061" y="-435"></use></g><use href="#MJMAIN-29" x="8707" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-19">
\mathrm{y}_2=g(VA_2+V'A_2')
</script><p></p><p data-anchor-id="bxby"><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-20-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -736.2000693679926 1204.406943983867 906.1072455171717" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1061" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-20">A_2</script>和<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-21-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -736.2000693679926 1204.406943983867 1063.304268568599" style="width: 2.782ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1061" y="445"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1061" y="-435"></use></g></svg></span><script type="math/tex" id="MathJax-Element-21">A_2'</script>则分别计算：</p><div class="md-section-divider"></div><p data-anchor-id="duw7"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-22-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1510.8662076095848 44270.673387096766 2521.7324152191695" style="width: 102.823ex; height: 5.806ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17439,0)"><g transform="translate(-15,0)"><g transform="translate(0,691)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1061" y="-213"></use></g><g transform="translate(0,-669)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1061" y="445"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1061" y="-435"></use></g></g><g transform="translate(1468,0)"><g transform="translate(0,691)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><use href="#MJMATHI-57" x="1996" y="0"></use><g transform="translate(3044,0)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1061" y="-213"></use></g><use href="#MJMAIN-2B" x="4471" y="0"></use><use href="#MJMATHI-55" x="5472" y="0"></use><g transform="translate(6239,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="747" y="-213"></use></g><use href="#MJMAIN-29" x="7222" y="0"></use></g><g transform="translate(0,-669)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1526" y="583"></use></g><g transform="translate(3370,0)"><use href="#MJMATHI-41"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1061" y="445"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="1061" y="-434"></use></g><use href="#MJMAIN-2B" x="4797" y="0"></use><g transform="translate(5798,0)"><use href="#MJMATHI-55"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1121" y="583"></use></g><g transform="translate(6885,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="747" y="-213"></use></g><use href="#MJMAIN-29" x="7868" y="0"></use></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,691)"><g id="mjx-eqn-8"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-669)"><g id="mjx-eqn-9"><use href="#MJMAIN-28"></use><use href="#MJMAIN-39" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-22">
\begin{align}
A_2&=f(WA_1+U\mathrm{x}_2)\\
A_2'&=f(W'A_3'+U'\mathrm{x}_2)\\
\end{align}
</script><p></p><p data-anchor-id="da2m">现在，我们已经可以看出一般的规律：正向计算时，隐藏层的值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-23-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 750.1191013989369 645.8854201102238" style="width: 1.694ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-23">\mathrm{s}_t</script>与<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-24-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 1654.5086745365313 645.8854201102238" style="width: 3.871ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-24">\mathrm{s}_{t-1}</script>有关；反向计算时，隐藏层的值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-25-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -730.6316888038108 750.1191013989369 1037.2297913500072" style="width: 1.694ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="557" y="445"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-25">\mathrm{s}_t'</script>与<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-26-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -730.6316888038108 1654.5086745365313 1115.718644061714" style="width: 3.871ex; height: 2.54ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="557" y="445"></use><g transform="translate(394,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2B" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-26">\mathrm{s}_{t+1}'</script>有关；最终的输出取决于正向和反向计算的<strong>加和</strong>。现在，我们仿照<strong>式1</strong>和<strong>式2</strong>，写出双向循环神经网络的计算方法：</p><div class="md-section-divider"></div><p data-anchor-id="c4ru"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-27-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2206.421433147703 44270.673387096766 3912.8428662954057" style="width: 102.823ex; height: 9.073ex; vertical-align: -4.113ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17437,0)"><g transform="translate(-15,0)"><g transform="translate(0,1377)"><use href="#MJMAIN-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-213"></use></g><g transform="translate(106,39)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g><g transform="translate(106,-1322)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="557" y="445"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g></g><g transform="translate(1119,0)"><g transform="translate(0,1377)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-67" x="1056" y="0"></use><use href="#MJMAIN-28" x="1536" y="0"></use><use href="#MJMATHI-56" x="1926" y="0"></use><g transform="translate(2695,0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g><use href="#MJMAIN-2B" x="3668" y="0"></use><g transform="translate(4668,0)"><use href="#MJMATHI-56"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1167" y="583"></use></g><g transform="translate(5788,0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="557" y="445"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g><use href="#MJMAIN-29" x="6539" y="0"></use></g><g transform="translate(0,39)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><use href="#MJMATHI-55" x="1996" y="0"></use><g transform="translate(2763,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="3870" y="0"></use><use href="#MJMATHI-57" x="4870" y="0"></use><g transform="translate(5919,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="7573" y="0"></use></g><g transform="translate(0,-1322)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-55"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1121" y="583"></use></g><g transform="translate(3083,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="4190" y="0"></use><g transform="translate(5190,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="1526" y="583"></use></g><g transform="translate(6565,0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="557" y="445"></use><g transform="translate(394,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2B" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="8219" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,1377)"><g id="mjx-eqn-10"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,39)"><g id="mjx-eqn-11"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1322)"><g id="mjx-eqn-12"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-27">
\begin{align}
\mathrm{o}_t&=g(V\mathrm{s}_t+V'\mathrm{s}_t')\\
\mathrm{s}_t&=f(U\mathrm{x}_t+W\mathrm{s}_{t-1})\\
\mathrm{s}_t'&=f(U'\mathrm{x}_t+W'\mathrm{s}_{t+1}')\\
\end{align}
</script><p></p><p data-anchor-id="6i8e">从上面三个公式我们可以看到，正向计算和反向计算<strong>不共享权重</strong>，也就是说U和U'、W和W'、V和V'都是不同的<strong>权重矩阵</strong>。</p><div class="md-section-divider"></div><h3 data-anchor-id="dj50" id="深度循环神经网络">深度循环神经网络</h3><p data-anchor-id="ifkl">前面我们介绍的<strong>循环神经网络</strong>只有一个隐藏层，我们当然也可以堆叠两个以上的隐藏层，这样就得到了<strong>深度循环神经网络</strong>。如下图所示：</p><p data-anchor-id="9vu6"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-df137de8007c3d26.png" alt="" title=""></p><p data-anchor-id="pw8b">我们把第i个隐藏层的值表示为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-28-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1072.1642641169133 1289.6415754442728 1378.76236666311" style="width: 3.024ex; height: 3.145ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-28">\mathrm{s}_t^{(i)}</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-29-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1072.1642641169133 1484.4494936611666 1378.76236666311" style="width: 3.508ex; height: 3.145ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-29">\mathrm{s}_t'^{(i)}</script>，则<strong>深度循环神经网络</strong>的计算方式可以表示为：</p><div class="md-section-divider"></div><p data-anchor-id="alk4"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-30-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 21.774ex; vertical-align: -10.282ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -4949.594491897511 44270.673387096766 9399.188983795022" style="width: 102.823ex; height: 21.774ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(15521,0)"><g transform="translate(-15,0)"><g transform="translate(737,3877)"><use href="#MJMAIN-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-213"></use></g><g transform="translate(304,2237)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g><g transform="translate(109,596)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g><g transform="translate(425,-821)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(194,-2425)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g><g transform="translate(0,-4065)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g></g><g transform="translate(1857,0)"><g transform="translate(0,3877)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-67" x="1056" y="0"></use><use href="#MJMAIN-28" x="1536" y="0"></use><g transform="translate(1926,0)"><use href="#MJMATHI-56"></use><g transform="translate(825,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><g transform="translate(3646,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g><use href="#MJMAIN-2B" x="5158" y="0"></use><g transform="translate(6159,0)"><use href="#MJMATHI-56"></use><g transform="translate(825,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g></g><g transform="translate(8074,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g><use href="#MJMAIN-29" x="9559" y="0"></use></g><g transform="translate(0,2237)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-55"></use><g transform="translate(792,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><g transform="translate(3684,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="734" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1513" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="2014" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g><use href="#MJMAIN-2B" x="6100" y="0"></use><g transform="translate(7101,0)"><use href="#MJMATHI-57"></use><g transform="translate(1079,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><g transform="translate(9076,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="10730" y="0"></use></g><g transform="translate(0,596)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-55"></use><g transform="translate(792,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g></g><g transform="translate(3878,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="734" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1513" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="2014" y="0"></use></g></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-395"></use></g><use href="#MJMAIN-2B" x="6490" y="0"></use><g transform="translate(7490,0)"><use href="#MJMATHI-57"></use><g transform="translate(1079,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g></g><g transform="translate(9660,0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="557" y="445"></use><g transform="translate(394,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2B" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="11314" y="0"></use></g><g transform="translate(0,-2425)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-55"></use><g transform="translate(792,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(3793,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="4900" y="0"></use><g transform="translate(5900,0)"><use href="#MJMATHI-57"></use><g transform="translate(1079,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(7985,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="9639" y="0"></use></g><g transform="translate(0,-4065)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-55"></use><g transform="translate(792,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g></g><g transform="translate(3988,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="5094" y="0"></use><g transform="translate(6095,0)"><use href="#MJMATHI-57"></use><g transform="translate(1079,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g></g><g transform="translate(8374,0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="557" y="445"></use><g transform="translate(394,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2B" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="10029" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,3877)"><g id="mjx-eqn-13"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,2237)"><g id="mjx-eqn-14"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,596)"><g id="mjx-eqn-15"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-821)"><g id="mjx-eqn-16"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2425)"><g id="mjx-eqn-17"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-4065)"><g id="mjx-eqn-18"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-30">
\begin{align}
\mathrm{o}_t&=g(V^{(i)}\mathrm{s}_t^{(i)}+V'^{(i)}\mathrm{s}_t'^{(i)})\\
\mathrm{s}_t^{(i)}&=f(U^{(i)}\mathrm{s}_t^{(i-1)}+W^{(i)}\mathrm{s}_{t-1})\\
\mathrm{s}_t'^{(i)}&=f(U'^{(i)}\mathrm{s}_t'^{(i-1)}+W'^{(i)}\mathrm{s}_{t+1}')\\
...\\
\mathrm{s}_t^{(1)}&=f(U^{(1)}\mathrm{x}_t+W^{(1)}\mathrm{s}_{t-1})\\
\mathrm{s}_t'^{(1)}&=f(U'^{(1)}\mathrm{x}_t+W'^{(1)}\mathrm{s}_{t+1}')\\
\end{align}
</script><p></p><div class="md-section-divider"></div><h2 data-anchor-id="x4ve" id="循环神经网络的训练">循环神经网络的训练</h2><div class="md-section-divider"></div><h3 data-anchor-id="jx75" id="循环神经网络的训练算法bptt">循环神经网络的训练算法：BPTT</h3><p data-anchor-id="4wrm">BPTT算法是针对<strong>循环层</strong>的训练算法，它的基本原理和BP算法是一样的，也包含同样的三个步骤：</p><ol data-anchor-id="xvrf">
<li>前向计算每个神经元的输出值；</li>
<li>反向计算每个神经元的<strong>误差项</strong><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-31-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 836.1815472394509 1051.3570288792273" style="width: 1.935ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-31">\delta_j</script>值，它是误差函数E对神经元j的<strong>加权输入</strong><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-32-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1820.181547239451 960.3570288792274" style="width: 4.234ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-32">net_j</script>的偏导数；</li>
<li>计算每个权重的梯度。</li>
</ol><p data-anchor-id="ml1w">最后再用<strong>随机梯度下降</strong>算法更新权重。</p><p data-anchor-id="v0mn">循环层如下图所示：</p><p data-anchor-id="885l"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-3b20294694c3904b.png" alt="" title=""></p><div class="md-section-divider"></div><h4 data-anchor-id="vgoz" id="前向计算">前向计算</h4><p data-anchor-id="gxrm">使用前面的<strong>式2</strong>对循环层进行前向计算：</p><div class="md-section-divider"></div><p data-anchor-id="ohio"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-33-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 8991.246877334406 1040.4001387359851" style="width: 20.927ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use><use href="#MJMAIN-3D" x="1027" y="0"></use><use href="#MJMATHI-66" x="2084" y="0"></use><use href="#MJMAIN-28" x="2634" y="0"></use><use href="#MJMATHI-55" x="3024" y="0"></use><g transform="translate(3791,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="4898" y="0"></use><use href="#MJMATHI-57" x="5898" y="0"></use><g transform="translate(6947,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="8601" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-33">
\mathrm{s}_t=f(U\mathrm{x}_t+W\mathrm{s}_{t-1})
</script><p></p><p data-anchor-id="4to7">注意，上面的<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-34-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 750.1191013989369 645.8854201102238" style="width: 1.694ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-34">\mathrm{s}_t</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-35-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 884.1191013989369 628.8854201102238" style="width: 2.056ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-35">\mathrm{x}_t</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-36-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 1654.5086745365313 645.8854201102238" style="width: 3.871ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-36">\mathrm{s}_{t-1}</script>都是向量，用<strong>黑体字母</strong>表示；而U、V是<strong>矩阵</strong>，用大写字母表示。<strong>向量的下标</strong>表示<strong>时刻</strong>，例如，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-37-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 750.1191013989369 645.8854201102238" style="width: 1.694ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="557" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-37">\mathrm{s}_t</script>表示在t时刻向量s的值。</p><p data-anchor-id="g015">我们假设输入向量x的维度是m，输出向量s的维度是n，则矩阵U的维度是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-38-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -511.20006936799257 2701.9444444444443 542.4001387359851" style="width: 6.29ex; height: 1.21ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMAIN-D7" x="822" y="0"></use><use href="#MJMATHI-6D" x="1823" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-38">n\times m</script>，矩阵W的维度是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-39-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -511.20006936799257 2423.9444444444443 542.4001387359851" style="width: 5.685ex; height: 1.21ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMAIN-D7" x="822" y="0"></use><use href="#MJMATHI-6E" x="1823" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-39">n\times n</script>。下面是上式展开成矩阵的样子，看起来更直观一些：</p><div class="md-section-divider"></div><p data-anchor-id="h9b8"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-40-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3360.505190922798 44270.673387096766 6221.010381845596" style="width: 102.823ex; height: 14.395ex; vertical-align: -6.774ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(8147,0)"><g transform="translate(-15,0)"><g transform="translate(0,3302)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4348.880303800534) scale(1,4.30825921028281)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5460"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2502)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="663" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><g transform="translate(35,997)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="663" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-2E" x="357" y="-203"></use><use href="#MJMAIN-2E" x="357" y="-1403"></use><g transform="translate(0,-2603)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="663" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g><g transform="translate(1981,3302)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4348.880303800534) scale(1,4.30825921028281)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5460"></use></g><use href="#MJMAIN-3D" x="2926" y="0"></use><use href="#MJMATHI-66" x="3982" y="0"></use><use href="#MJMAIN-28" x="4533" y="0"></use><g transform="translate(4922,0)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(106,2350)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><g transform="translate(1380,0)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2E" x="2760" y="0"></use><use href="#MJMAIN-2E" x="3205" y="0"></use><use href="#MJMAIN-2E" x="3650" y="0"></use><g transform="translate(4096,0)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="500" y="0"></use></g></g></g><g transform="translate(106,1150)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><g transform="translate(1380,0)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2E" x="2760" y="0"></use><use href="#MJMAIN-2E" x="3205" y="0"></use><use href="#MJMAIN-2E" x="3650" y="0"></use><g transform="translate(4096,0)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="500" y="0"></use></g></g></g><use href="#MJMAIN-2E" x="2838" y="-50"></use><use href="#MJMAIN-2E" x="2838" y="-1250"></use><g transform="translate(0,-2450)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g><g transform="translate(1451,0)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g><use href="#MJMAIN-2E" x="2902" y="0"></use><use href="#MJMAIN-2E" x="3347" y="0"></use><use href="#MJMAIN-2E" x="3792" y="0"></use><g transform="translate(4237,0)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="600" y="0"></use></g></g></g></g></g><g transform="translate(6942,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g><g transform="translate(12533,0)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(133,2350)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><g transform="translate(133,1150)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><use href="#MJMAIN-2E" x="507" y="-50"></use><use href="#MJMAIN-2E" x="507" y="-1250"></use><g transform="translate(0,-2450)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="809" y="-213"></use></g></g></g><g transform="translate(2280,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g><use href="#MJMAIN-2B" x="15703" y="0"></use><g transform="translate(16704,0)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(106,2350)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><g transform="translate(1524,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2E" x="3048" y="0"></use><use href="#MJMAIN-2E" x="3493" y="0"></use><use href="#MJMAIN-2E" x="3938" y="0"></use><g transform="translate(4384,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g><g transform="translate(106,1150)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><g transform="translate(1524,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2E" x="3048" y="0"></use><use href="#MJMAIN-2E" x="3493" y="0"></use><use href="#MJMAIN-2E" x="3938" y="0"></use><g transform="translate(4384,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g><use href="#MJMAIN-2E" x="2956" y="-50"></use><use href="#MJMAIN-2E" x="2956" y="-1250"></use><g transform="translate(0,-2450)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g><g transform="translate(1595,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g><use href="#MJMAIN-2E" x="3190" y="0"></use><use href="#MJMAIN-2E" x="3635" y="0"></use><use href="#MJMAIN-2E" x="4080" y="0"></use><g transform="translate(4525,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="600" y="0"></use></g></g></g></g></g><g transform="translate(7178,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g><g transform="translate(24550,0)"><g transform="translate(0,3341)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4427.172146680622) scale(1,4.438096428989423)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5538"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2516)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><g transform="translate(0,984)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-2E" x="725" y="-216"></use><use href="#MJMAIN-2E" x="725" y="-1416"></use><g transform="translate(0,-2642)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g><g transform="translate(2716,3341)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4427.172146680622) scale(1,4.438096428989423)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5538"></use></g></g><use href="#MJMAIN-29" x="27934" y="0"></use></g></g><g transform="translate(41690,0)"><g id="mjx-eqn-19"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-40">
\begin{align}
\begin{bmatrix}
s_1^t\\
s_2^t\\
.\\.\\
s_n^t\\
\end{bmatrix}=f(
\begin{bmatrix}
u_{11} u_{12} ... u_{1m}\\
u_{21} u_{22} ... u_{2m}\\
.\\.\\
u_{n1} u_{n2} ... u_{nm}\\
\end{bmatrix}
\begin{bmatrix}
x_1\\
x_2\\
.\\.\\
x_m\\
\end{bmatrix}+
\begin{bmatrix}
w_{11} w_{12} ... w_{1n}\\
w_{21} w_{22} ... w_{2n}\\
.\\.\\
w_{n1} w_{n2} ... w_{nn}\\
\end{bmatrix}
\begin{bmatrix}
s_1^{t-1}\\
s_2^{t-1}\\
.\\.\\
s_n^{t-1}\\
\end{bmatrix})
\end{align}
</script><p></p><p data-anchor-id="y7y5">在这里我们用<strong>手写体字母</strong>表示向量的一个<strong>元素</strong>，它的下标表示它是这个向量的第几个元素，它的上标表示第几个<strong>时刻</strong>。例如，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-41-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -815.4845025461965 861.1815472394509 1283.302951202926" style="width: 2.056ex; height: 3.024ex; vertical-align: -1.21ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="663" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="663" y="-430"></use></g></svg></span><script type="math/tex" id="MathJax-Element-41">s_j^t</script>表示向量s的第j个元素在t时刻的值。<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-42-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 1208.486940139403 776.3570288792274" style="width: 2.782ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-42">u_{ji}</script>表示<strong>输入层</strong>第i个神经元到<strong>循环层</strong>第j个神经元的权重。<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-43-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-43">w_{ji}</script>表示<strong>循环层</strong>第t-1时刻的第i个神经元到<strong>循环层</strong>第t个时刻的第j个神经元的权重。</p><div class="md-section-divider"></div><h4 data-anchor-id="73ed" id="误差项的计算">误差项的计算</h4><p data-anchor-id="5y27">BTPP算法将第l层t时刻的<strong>误差项</strong><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-44-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -864.9819772292548 800.1191013989369 1171.5800797754512" style="width: 1.815ex; height: 2.661ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="641" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-44">\delta_t^l</script>值沿两个方向传播，一个方向是其传递到上一层网络，得到<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-45-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -864.9819772292548 1669.110947321779 1171.5800797754512" style="width: 3.871ex; height: 2.661ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-45">\delta_t^{l-1}</script>，这部分只和权重矩阵U有关；另一个是方向是将其沿时间线传递到初始<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-46-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 815.406943983867 816.1072455171717" style="width: 1.935ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-46">t_1</script>时刻，得到<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-47-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -864.9819772292548 898.406943983867 1192.086176429861" style="width: 2.056ex; height: 2.782ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="641" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use></g></svg></span><script type="math/tex" id="MathJax-Element-47">\delta_1^l</script>，这部分只和权重矩阵W有关。</p><p data-anchor-id="3txs">我们用向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-48-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -635.2000693679926 1746.119101398937 812.8854201102238" style="width: 4.113ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-48">\mathrm{net}_t</script>表示神经元在t时刻的<strong>加权输入</strong>，因为：</p><div class="md-section-divider"></div><p data-anchor-id="tqfx"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-49-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1445.7000693679925 44270.673387096766 2391.400138735985" style="width: 102.823ex; height: 5.565ex; vertical-align: -2.298ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17980,0)"><g transform="translate(-15,0)"><g transform="translate(0,626)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g><g transform="translate(91,-676)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(2009,0)"><g transform="translate(0,626)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-55" x="1056" y="0"></use><g transform="translate(1823,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="2930" y="0"></use><use href="#MJMATHI-57" x="3930" y="0"></use><g transform="translate(4979,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(0,-676)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="4646" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,626)"><g id="mjx-eqn-20"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-676)"><g id="mjx-eqn-21"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-49">
\begin{align}
\mathrm{net}_t&=U\mathrm{x}_t+W\mathrm{s}_{t-1}\\
\mathrm{s}_{t-1}&=f(\mathrm{net}_{t-1})\\
\end{align}
</script><p></p><p data-anchor-id="p0e6">因此：</p><div class="md-section-divider"></div><p data-anchor-id="zfra"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-50-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1387.8006415067248 44270.673387096766 2275.6012830134496" style="width: 102.823ex; height: 5.323ex; vertical-align: -2.177ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(16644,0)"><g transform="translate(-15,0)"><g transform="translate(0,-24)"><g transform="translate(120,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(512,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g></g></g><g transform="translate(3841,0)"><g transform="translate(0,-24)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(105,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g><g transform="translate(4016,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(558,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,-24)"><g id="mjx-eqn-22"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-50">
\begin{align}
\frac{\partial{\mathrm{net}_t}}{\partial{\mathrm{net}_{t-1}}}&=\frac{\partial{\mathrm{net}_t}}{\partial{\mathrm{s}_{t-1}}}\frac{\partial{\mathrm{s}_{t-1}}}{\partial{\mathrm{net}_{t-1}}}\\
\end{align}
</script><p></p><p data-anchor-id="5r3s">我们用a表示列向量，用<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-51-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -861.5104670124718 1098.6567273459227 892.7105363804644" style="width: 2.54ex; height: 2.056ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="707" y="513"></use></g></svg></span><script type="math/tex" id="MathJax-Element-51">\mathrm{a}^T</script>表示行向量。上式的第一项是向量函数对向量求导，其结果为Jacobian矩阵：</p><div class="md-section-divider"></div><p data-anchor-id="c9zu"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-52-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -9196.013757613397 44270.673387096766 17892.027515226793" style="width: 102.823ex; height: 41.613ex; vertical-align: -20.323ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(13072,0)"><g transform="translate(-15,0)"><g transform="translate(0,3701)"><g transform="translate(120,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(105,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g></g></g><g transform="translate(2937,0)"><g transform="translate(0,3701)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,5475)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-8689.121370900592) scale(1,11.506005590216569)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-9807"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,3901)"><g transform="translate(120,0)"><rect stroke="none" width="2569" height="60" x="0" y="220"></rect><g transform="translate(60,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g></g></g><g transform="translate(136,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g></g></g><g transform="translate(35,824)"><g transform="translate(120,0)"><rect stroke="none" width="2569" height="60" x="0" y="220"></rect><g transform="translate(60,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g></g></g><g transform="translate(136,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g></g></g><g transform="translate(0,-3874)"><g transform="translate(120,0)"><rect stroke="none" width="2640" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g></g></g><g transform="translate(171,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g></g></g></g><g transform="translate(3866,0)"><g transform="translate(35,3901)"><g transform="translate(120,0)"><rect stroke="none" width="2569" height="60" x="0" y="220"></rect><g transform="translate(60,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g></g></g><g transform="translate(136,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g></g></g><g transform="translate(35,824)"><g transform="translate(120,0)"><rect stroke="none" width="2569" height="60" x="0" y="220"></rect><g transform="translate(60,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g></g></g><g transform="translate(136,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g></g></g><use href="#MJMAIN-2E" x="1301" y="-800"></use><use href="#MJMAIN-2E" x="1301" y="-2000"></use><g transform="translate(0,-3874)"><g transform="translate(120,0)"><rect stroke="none" width="2640" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g></g></g><g transform="translate(171,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g></g></g></g><g transform="translate(7746,0)"><g transform="translate(0,3901)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,824)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-3874)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(9915,0)"><g transform="translate(35,3901)"><g transform="translate(120,0)"><rect stroke="none" width="2569" height="60" x="0" y="220"></rect><g transform="translate(60,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g></g></g><g transform="translate(136,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g></g><g transform="translate(35,824)"><g transform="translate(120,0)"><rect stroke="none" width="2569" height="60" x="0" y="220"></rect><g transform="translate(60,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g></g></g><g transform="translate(136,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g></g><g transform="translate(0,-3874)"><g transform="translate(120,0)"><rect stroke="none" width="2640" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g></g></g><g transform="translate(171,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g></g></g></g><g transform="translate(13797,5475)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-8689.121370900592) scale(1,11.506005590216569)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-9807"></use></g></g></g><g transform="translate(0,-4725)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2350)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(35,1150)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(0,-2450)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g></g></g><g transform="translate(2580,0)"><g transform="translate(35,2350)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(35,1150)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2E" x="658" y="-50"></use><use href="#MJMAIN-2E" x="658" y="-1250"></use><g transform="translate(0,-2450)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g></g><g transform="translate(5175,0)"><g transform="translate(0,2350)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1150)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2450)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(7344,0)"><g transform="translate(35,2350)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g><g transform="translate(35,1150)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g><g transform="translate(0,-2450)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="600" y="0"></use></g></g></g></g><g transform="translate(10011,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g></g><g transform="translate(0,-8426)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-57" x="1056" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,3701)"><g id="mjx-eqn-23"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-4725)"><g id="mjx-eqn-24"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-8426)"><g id="mjx-eqn-25"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-52">
\begin{align}
\frac{\partial{\mathrm{net}_t}}{\partial{\mathrm{s}_{t-1}}}&=
\begin{bmatrix}
\frac{\partial{net_1^t}}{\partial{s_1^{t-1}}}& \frac{\partial{net_1^t}}{\partial{s_2^{t-1}}}& ...&  \frac{\partial{net_1^t}}{\partial{s_n^{t-1}}}\\
\frac{\partial{net_2^t}}{\partial{s_1^{t-1}}}& \frac{\partial{net_2^t}}{\partial{s_2^{t-1}}}& ...&  \frac{\partial{net_2^t}}{\partial{s_n^{t-1}}}\\
&.\\&.\\
\frac{\partial{net_n^t}}{\partial{s_1^{t-1}}}& \frac{\partial{net_n^t}}{\partial{s_2^{t-1}}}& ...&  \frac{\partial{net_n^t}}{\partial{s_n^{t-1}}}\\
\end{bmatrix}\\
&=\begin{bmatrix}
w_{11} & w_{12} & ... & w_{1n}\\
w_{21} & w_{22} & ... & w_{2n}\\
&.\\&.\\
w_{n1} & w_{n2} & ... & w_{nn}\\
\end{bmatrix}\\
&=W
\end{align}
</script><p></p><p data-anchor-id="mt0d">同理，上式第二项也是一个Jacobian矩阵：</p><div class="md-section-divider"></div><p data-anchor-id="80lk"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-53-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -9484.534738162223 44270.673387096766 18469.069476324446" style="width: 102.823ex; height: 42.944ex; vertical-align: -20.927ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(10445,0)"><g transform="translate(-15,0)"><g transform="translate(0,3947)"><g transform="translate(120,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(558,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g></g></g><g transform="translate(3841,0)"><g transform="translate(0,3947)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,5518)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-8773.833699852014) scale(1,11.646490381180786)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-9892"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,3915)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g></g></g></g></g><g transform="translate(0,810)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g></g></g></g></g><g transform="translate(0,-3916)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g></g></g></g></g></g><g transform="translate(4601,0)"><g transform="translate(0,3915)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g></g></g></g></g><g transform="translate(0,810)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g></g></g></g></g><use href="#MJMAIN-2E" x="1668" y="-814"></use><use href="#MJMAIN-2E" x="1668" y="-2014"></use><g transform="translate(0,-3916)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g></g></g></g></g></g><g transform="translate(9217,0)"><g transform="translate(0,3915)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,810)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-3916)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(11386,0)"><g transform="translate(0,3915)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g></g></g></g></g><g transform="translate(0,810)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,778)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g></g></g></g></g><g transform="translate(0,-3916)"><g transform="translate(120,0)"><rect stroke="none" width="3376" height="60" x="0" y="220"></rect><g transform="translate(539,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(60,-795)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g></g></g></g></g></g></g><g transform="translate(16003,5518)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-8773.833699852014) scale(1,11.646490381180786)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-9892"></use></g></g></g><g transform="translate(0,-4738)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,3366)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4478.087709594529) scale(1,4.522533515082138)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5589"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2542)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use><use href="#MJMAIN-28" x="863" y="0"></use><use href="#MJMATHI-6E" x="1252" y="0"></use><use href="#MJMATHI-65" x="1853" y="0"></use><g transform="translate(2319,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g><use href="#MJMAIN-29" x="3941" y="0"></use></g><use href="#MJMAIN-30" x="1915" y="1009"></use><use href="#MJMAIN-30" x="1915" y="-2616"></use></g><g transform="translate(5316,0)"><use href="#MJMAIN-30" x="1915" y="2542"></use><g transform="translate(0,1009)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use><use href="#MJMAIN-28" x="863" y="0"></use><use href="#MJMATHI-6E" x="1252" y="0"></use><use href="#MJMATHI-65" x="1853" y="0"></use><g transform="translate(2319,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g><use href="#MJMAIN-29" x="3941" y="0"></use></g><use href="#MJMAIN-2E" x="2026" y="-191"></use><use href="#MJMAIN-2E" x="2026" y="-1391"></use><use href="#MJMAIN-30" x="1915" y="-2616"></use></g><g transform="translate(10647,0)"><g transform="translate(0,2542)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1009)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2616)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(12816,0)"><use href="#MJMAIN-30" x="1915" y="2542"></use><use href="#MJMAIN-30" x="1915" y="1009"></use><g transform="translate(0,-2616)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use><use href="#MJMAIN-28" x="863" y="0"></use><use href="#MJMATHI-6E" x="1252" y="0"></use><use href="#MJMATHI-65" x="1853" y="0"></use><g transform="translate(2319,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g><use href="#MJMAIN-29" x="3941" y="0"></use></g></g></g><g transform="translate(18148,3366)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4478.087709594529) scale(1,4.522533515082138)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5589"></use></g></g></g><g transform="translate(0,-8715)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-64" x="1056" y="0"></use><use href="#MJMATHI-69" x="1579" y="0"></use><use href="#MJMATHI-61" x="1925" y="0"></use><use href="#MJMATHI-67" x="2454" y="0"></use><use href="#MJMAIN-5B" x="2935" y="0"></use><g transform="translate(3213,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="4077" y="0"></use><g transform="translate(4466,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="7117" y="0"></use><use href="#MJMAIN-5D" x="7506" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,3947)"><g id="mjx-eqn-26"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-4738)"><g id="mjx-eqn-27"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-8715)"><g id="mjx-eqn-28"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-53">
\begin{align}
\frac{\partial{\mathrm{s}_{t-1}}}{\partial{\mathrm{net}_{t-1}}}&=
\begin{bmatrix}
\frac{\partial{s_1^{t-1}}}{\partial{net_1^{t-1}}}& \frac{\partial{s_1^{t-1}}}{\partial{net_2^{t-1}}}& ...&  \frac{\partial{s_1^{t-1}}}{\partial{net_n^{t-1}}}\\
\frac{\partial{s_2^{t-1}}}{\partial{net_1^{t-1}}}& \frac{\partial{s_2^{t-1}}}{\partial{net_2^{t-1}}}& ...&  \frac{\partial{s_2^{t-1}}}{\partial{net_n^{t-1}}}\\
&.\\&.\\
\frac{\partial{s_n^{t-1}}}{\partial{net_1^{t-1}}}& \frac{\partial{s_n^{t-1}}}{\partial{net_2^{t-1}}}& ...&  \frac{\partial{s_n^{t-1}}}{\partial{net_n^{t-1}}}\\
\end{bmatrix}\\
&=\begin{bmatrix}
f'(net_1^{t-1}) & 0 & ... & 0\\
0 & f'(net_2^{t-1}) & ... & 0\\
&.\\&.\\
0 & 0 & ... & f'(net_n^{t-1})\\
\end{bmatrix}\\
&=diag[f'(\mathrm{net}_{t-1})]
\end{align}
</script><p></p><p data-anchor-id="6zzs">其中，diag[a]表示根据向量a创建一个对角矩阵，即</p><div class="md-section-divider"></div><p data-anchor-id="2md8"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true"><span class="MathJax_SVG" id="MathJax-Element-54-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3178.053622758586 44270.673387096766 6286.661245517172" style="width: 102.823ex; height: 14.637ex; vertical-align: -7.258ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(15466,0)"><use href="#MJMATHI-64"></use><use href="#MJMATHI-69" x="523" y="0"></use><use href="#MJMATHI-61" x="869" y="0"></use><use href="#MJMATHI-67" x="1398" y="0"></use><use href="#MJMAIN-28" x="1879" y="0"></use><use href="#MJMAIN-61" x="2268" y="0"></use><use href="#MJMAIN-29" x="2769" y="0"></use><use href="#MJMAIN-3D" x="3436" y="0"></use><g transform="translate(4492,0)"><g transform="translate(0,3158)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4062.8731546176414) scale(1,3.833952163545011)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5173"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2358)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="748" y="-213"></use></g><use href="#MJMAIN-30" x="241" y="1141"></use><use href="#MJMAIN-30" x="241" y="-2459"></use></g><g transform="translate(1969,0)"><use href="#MJMAIN-30" x="241" y="2358"></use><g transform="translate(0,1141)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="748" y="-213"></use></g><use href="#MJMAIN-2E" x="352" y="-59"></use><use href="#MJMAIN-2E" x="352" y="-1259"></use><use href="#MJMAIN-30" x="241" y="-2459"></use></g><g transform="translate(3952,0)"><g transform="translate(0,2358)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1141)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2459)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(6121,0)"><use href="#MJMAIN-30" x="276" y="2358"></use><use href="#MJMAIN-30" x="276" y="1141"></use><g transform="translate(0,-2459)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="748" y="-213"></use></g></g></g><g transform="translate(8176,3158)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4062.8731546176414) scale(1,3.833952163545011)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5173"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-54">
diag(\mathrm{a})=\begin{bmatrix}
a_1 & 0 & ... & 0\\
0 & a_2 & ... & 0\\
&.\\&.\\
0 & 0 & ... & a_n\\
\end{bmatrix}\\
</script><p></p><p data-anchor-id="0p9q">最后，将两项合在一起，可得：</p><div class="md-section-divider"></div><p data-anchor-id="avov"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-55-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -5363.347280100665 44270.673387096766 10226.69456020133" style="width: 102.823ex; height: 23.71ex; vertical-align: -11.371ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(8017,0)"><g transform="translate(-15,0)"><g transform="translate(0,3951)"><g transform="translate(120,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(512,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g></g></g><g transform="translate(3841,0)"><g transform="translate(0,3951)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(105,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g><g transform="translate(4016,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(558,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g></g><g transform="translate(0,1997)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-57" x="1056" y="0"></use><use href="#MJMATHI-64" x="2104" y="0"></use><use href="#MJMATHI-69" x="2628" y="0"></use><use href="#MJMATHI-61" x="2973" y="0"></use><use href="#MJMATHI-67" x="3503" y="0"></use><use href="#MJMAIN-5B" x="3983" y="0"></use><g transform="translate(4262,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="5125" y="0"></use><g transform="translate(5515,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="8165" y="0"></use><use href="#MJMAIN-5D" x="8555" y="0"></use></g><g transform="translate(0,-1949)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,3395)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5646"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><g transform="translate(1524,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="2387" y="0"></use><use href="#MJMATHI-6E" x="2777" y="0"></use><use href="#MJMATHI-65" x="3377" y="0"></use><g transform="translate(3844,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g><use href="#MJMAIN-29" x="5465" y="0"></use></g><g transform="translate(35,1038)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><g transform="translate(1524,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="2387" y="0"></use><use href="#MJMATHI-6E" x="2777" y="0"></use><use href="#MJMATHI-65" x="3377" y="0"></use><g transform="translate(3844,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g><use href="#MJMAIN-29" x="5465" y="0"></use></g><g transform="translate(0,-2588)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g><g transform="translate(1595,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="2458" y="0"></use><use href="#MJMATHI-6E" x="2847" y="0"></use><use href="#MJMATHI-65" x="3448" y="0"></use><g transform="translate(3914,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g><use href="#MJMAIN-29" x="5536" y="0"></use></g></g><g transform="translate(6911,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g><g transform="translate(1524,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="2387" y="0"></use><use href="#MJMATHI-6E" x="2777" y="0"></use><use href="#MJMATHI-65" x="3377" y="0"></use><g transform="translate(3844,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g><use href="#MJMAIN-29" x="5465" y="0"></use></g><g transform="translate(35,1038)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g><g transform="translate(1524,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="2387" y="0"></use><use href="#MJMATHI-6E" x="2777" y="0"></use><use href="#MJMATHI-65" x="3377" y="0"></use><g transform="translate(3844,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g><use href="#MJMAIN-29" x="5465" y="0"></use></g><use href="#MJMAIN-2E" x="2823" y="-162"></use><use href="#MJMAIN-2E" x="2823" y="-1362"></use><g transform="translate(0,-2588)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g><g transform="translate(1595,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="2458" y="0"></use><use href="#MJMATHI-6E" x="2847" y="0"></use><use href="#MJMATHI-65" x="3448" y="0"></use><g transform="translate(3914,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g><use href="#MJMAIN-29" x="5536" y="0"></use></g></g><g transform="translate(13837,0)"><g transform="translate(0,2570)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1038)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2588)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(16006,0)"><g transform="translate(191,2570)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g><use href="#MJMATHI-66" x="1595" y="0"></use><use href="#MJMAIN-28" x="2145" y="0"></use><use href="#MJMATHI-6E" x="2535" y="0"></use><use href="#MJMATHI-65" x="3135" y="0"></use><g transform="translate(3602,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g><use href="#MJMAIN-29" x="5223" y="0"></use></g><g transform="translate(191,1038)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g><use href="#MJMATHI-66" x="1595" y="0"></use><use href="#MJMAIN-28" x="2145" y="0"></use><use href="#MJMATHI-6E" x="2535" y="0"></use><use href="#MJMATHI-65" x="3135" y="0"></use><g transform="translate(3602,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g><use href="#MJMAIN-29" x="5223" y="0"></use></g><g transform="translate(0,-2588)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="600" y="0"></use></g><g transform="translate(1665,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="2529" y="0"></use><use href="#MJMATHI-6E" x="2918" y="0"></use><use href="#MJMATHI-65" x="3519" y="0"></use><g transform="translate(3985,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g><use href="#MJMAIN-29" x="5607" y="0"></use></g></g></g><g transform="translate(23004,3395)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5646"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,3951)"><g id="mjx-eqn-29"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,1997)"><g id="mjx-eqn-30"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1949)"><g id="mjx-eqn-31"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-55">
\begin{align}
\frac{\partial{\mathrm{net}_t}}{\partial{\mathrm{net}_{t-1}}}&=\frac{\partial{\mathrm{net}_t}}{\partial{\mathrm{s}_{t-1}}}\frac{\partial{\mathrm{s}_{t-1}}}{\partial{\mathrm{net}_{t-1}}}\\
&=Wdiag[f'(\mathrm{net}_{t-1})]\\
&=\begin{bmatrix}
w_{11}f'(net_1^{t-1}) & w_{12}f'(net_2^{t-1}) & ... & w_{1n}f(net_n^{t-1})\\
w_{21}f'(net_1^{t-1}) & w_{22} f'(net_2^{t-1}) & ... & w_{2n}f(net_n^{t-1})\\
&.\\&.\\
w_{n1}f'(net_1^{t-1}) & w_{n2} f'(net_2^{t-1}) & ... & w_{nn} f'(net_n^{t-1})\\
\end{bmatrix}\\
\end{align}
</script><p></p><p data-anchor-id="fels">上式描述了将<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-56-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 451.5 767.4001387359851" style="width: 1.089ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use></g></svg></span><script type="math/tex" id="MathJax-Element-56">\delta</script>沿时间往前传递一个时刻的规律，有了这个规律，我们就可以求得任意时刻k的<strong>误差项</strong><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-57-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 913.2561863887845 914.8854201102238" style="width: 2.177ex; height: 2.177ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-57">\delta_k</script>：</p><div class="md-section-divider"></div><p data-anchor-id="tglh"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-58-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -6262.901977849935 44270.673387096766 12025.80395569987" style="width: 102.823ex; height: 27.944ex; vertical-align: -13.427ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(8778,0)"><g transform="translate(-15,0)"><g transform="translate(0,4851)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-463"></use><use href="#MJMAIN-3D" x="1329" y="0"></use></g><use href="#MJMAIN-3D" x="1329" y="2313"></use><use href="#MJMAIN-3D" x="1329" y="-226"></use><use href="#MJMAIN-3D" x="1329" y="-2217"></use><use href="#MJMAIN-3D" x="1329" y="-4527"></use></g><g transform="translate(2371,0)"><g transform="translate(0,4851)"><g transform="translate(120,0)"><rect stroke="none" width="2546" height="60" x="0" y="220"></rect><g transform="translate(607,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1966" y="-213"></use></g></g></g></g><g transform="translate(0,2313)"><g transform="translate(120,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(550,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g></g><g transform="translate(2960,0)"><rect stroke="none" width="2546" height="60" x="0" y="220"></rect><g transform="translate(116,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1966" y="-213"></use></g></g></g></g><g transform="translate(0,-226)"><g transform="translate(120,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(550,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g></g><g transform="translate(2960,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(512,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g><g transform="translate(6704,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1140" y="0"></use></g></g></g></g><use href="#MJMAIN-2E" x="10329" y="0"></use><use href="#MJMAIN-2E" x="10774" y="0"></use><use href="#MJMAIN-2E" x="11219" y="0"></use><g transform="translate(11785,0)"><rect stroke="none" width="3451" height="60" x="0" y="220"></rect><g transform="translate(60,678)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2B" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1300" y="0"></use></g></g></g><g transform="translate(512,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1966" y="-213"></use></g></g></g></g><g transform="translate(0,-2217)"><use href="#MJMATHI-57"></use><use href="#MJMATHI-64" x="1048" y="0"></use><use href="#MJMATHI-69" x="1572" y="0"></use><use href="#MJMATHI-61" x="1917" y="0"></use><use href="#MJMATHI-67" x="2447" y="0"></use><use href="#MJMAIN-5B" x="2927" y="0"></use><g transform="translate(3206,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="4069" y="0"></use><g transform="translate(4458,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="7109" y="0"></use><use href="#MJMAIN-5D" x="7498" y="0"></use><use href="#MJMATHI-57" x="7777" y="0"></use><use href="#MJMATHI-64" x="8825" y="0"></use><use href="#MJMATHI-69" x="9349" y="0"></use><use href="#MJMATHI-61" x="9694" y="0"></use><use href="#MJMATHI-67" x="10224" y="0"></use><use href="#MJMAIN-5B" x="10704" y="0"></use><g transform="translate(10983,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="11846" y="0"></use><g transform="translate(12236,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="14886" y="0"></use><use href="#MJMAIN-5D" x="15276" y="0"></use><use href="#MJMAIN-2E" x="15554" y="0"></use><use href="#MJMAIN-2E" x="15999" y="0"></use><use href="#MJMAIN-2E" x="16445" y="0"></use><use href="#MJMATHI-57" x="16890" y="0"></use><use href="#MJMATHI-64" x="17938" y="0"></use><use href="#MJMATHI-69" x="18462" y="0"></use><use href="#MJMATHI-61" x="18807" y="0"></use><use href="#MJMATHI-67" x="19337" y="0"></use><use href="#MJMAIN-5B" x="19817" y="0"></use><g transform="translate(20096,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="20959" y="0"></use><g transform="translate(21349,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1966" y="-213"></use></g><use href="#MJMAIN-29" x="23208" y="0"></use><use href="#MJMAIN-5D" x="23597" y="0"></use><g transform="translate(23876,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="641" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g></g><g transform="translate(0,-4527)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><g transform="translate(1218,0)"><use href="#MJSZ2-220F"></use><g transform="translate(57,-1110)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-69"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-3D" x="345" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1124" y="0"></use></g><g transform="translate(59,1151)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMATHI-57" x="2663" y="0"></use><use href="#MJMATHI-64" x="3712" y="0"></use><use href="#MJMATHI-69" x="4235" y="0"></use><use href="#MJMATHI-61" x="4581" y="0"></use><use href="#MJMATHI-67" x="5110" y="0"></use><use href="#MJMAIN-5B" x="5591" y="0"></use><g transform="translate(5869,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="6732" y="0"></use><g transform="translate(7122,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="1966" y="-213"></use></g><use href="#MJMAIN-29" x="8857" y="0"></use><use href="#MJMAIN-5D" x="9246" y="0"></use><use href="#MJMAIN-28" x="11525" y="0"></use><g transform="translate(11914,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-33" x="12748" y="0"></use><use href="#MJMAIN-29" x="13248" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,4851)"><g id="mjx-eqn-32"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,2313)"><g id="mjx-eqn-33"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-226)"><g id="mjx-eqn-34"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2217)"><g id="mjx-eqn-35"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-4527)"><g id="mjx-eqn-36"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-58">
\begin{align}
\delta_k^T=&\frac{\partial{E}}{\partial{\mathrm{net}_k}}\\
=&\frac{\partial{E}}{\partial{\mathrm{net}_t}}\frac{\partial{\mathrm{net}_t}}{\partial{\mathrm{net}_k}}\\
=&\frac{\partial{E}}{\partial{\mathrm{net}_t}}\frac{\partial{\mathrm{net}_t}}{\partial{\mathrm{net}_{t-1}}}\frac{\partial{\mathrm{net}_{t-1}}}{\partial{\mathrm{net}_{t-2}}}...\frac{\partial{\mathrm{net}_{k+1}}}{\partial{\mathrm{net}_{k}}}\\
=&Wdiag[f'(\mathrm{net}_{t-1})]
Wdiag[f'(\mathrm{net}_{t-2})]
...
Wdiag[f'(\mathrm{net}_{k})]
\delta_t^l\\
=&\delta_t^T\prod_{i=k}^{t-1}Wdiag[f'(\mathrm{net}_{i})]\qquad(式3)
\end{align}
</script><p></p><p data-anchor-id="wt0q"><strong>式3</strong>就是将误差项沿时间反向传播的算法。</p><p data-anchor-id="8fr9"><strong>循环层</strong>将<strong>误差项</strong>反向传递到上一层网络，与普通的<strong>全连接层</strong>是完全一样的，这在前面的文章<a href="https://www.zybuluo.com/hanbingtao/note/476663" target="_blank">零基础入门深度学习(3) - 神经网络和反向传播算法</a>中已经详细讲过了，在此仅简要描述一下。</p><p data-anchor-id="mctw"><strong>循环层</strong>的<strong>加权输入</strong><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-59-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -874.2383890738297 1701.5713741841846 905.4384584418223" style="width: 3.992ex; height: 2.056ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="1966" y="513"></use></g></svg></span><script type="math/tex" id="MathJax-Element-59">\mathrm{net}^l</script>与上一层的<strong>加权输入</strong><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-60-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -874.2383890738297 2605.960947321779 905.4384584418223" style="width: 6.048ex; height: 2.056ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,362)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-60">\mathrm{net}^{l-1}</script>关系如下：</p><div class="md-section-divider"></div><p data-anchor-id="j555"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-61-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1582.0082163297461 44270.673387096766 2664.0164326594922" style="width: 102.823ex; height: 6.169ex; vertical-align: -2.661ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17564,0)"><g transform="translate(-15,0)"><g transform="translate(0,717)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="1966" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use><use href="#MJMAIN-3D" x="2023" y="0"></use></g><g transform="translate(30,-776)"><use href="#MJMAIN-61"></use><g transform="translate(500,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-395"></use><use href="#MJMAIN-3D" x="1993" y="0"></use></g></g><g transform="translate(3065,0)"><g transform="translate(0,717)"><use href="#MJMATHI-55"></use><g transform="translate(767,0)"><use href="#MJMAIN-61"></use><g transform="translate(500,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-395"></use></g><use href="#MJMAIN-2B" x="2705" y="0"></use><use href="#MJMATHI-57" x="3706" y="0"></use><g transform="translate(4754,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(0,-776)"><use href="#MJMATHI-66"></use><g transform="translate(568,412)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use href="#MJMAIN-28" x="1784" y="0"></use><g transform="translate(2173,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g><use href="#MJMAIN-29" x="4779" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,717)"><g id="mjx-eqn-37"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-776)"><g id="mjx-eqn-38"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-61">
\begin{align}
\mathrm{net}_t^l=&U\mathrm{a}_t^{l-1}+W\mathrm{s}_{t-1}\\
\mathrm{a}_t^{l-1}=&f^{l-1}(\mathrm{net}_t^{l-1})
\end{align}
</script><p></p><p data-anchor-id="p818">上式中<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-62-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -864.9819772292548 1746.119101398937 1171.5800797754512" style="width: 4.113ex; height: 2.661ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="1966" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-62">\mathrm{net}_t^l</script>是第l层神经元的<strong>加权输入</strong>(假设第l层是<strong>循环层</strong>)；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-63-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -864.9819772292548 2605.960947321779 1171.5800797754512" style="width: 6.048ex; height: 2.661ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-63">\mathrm{net}_t^{l-1}</script>是第l-1层神经元的<strong>加权输入</strong>；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-64-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -864.9819772292548 1715.9609473217788 1171.5800797754514" style="width: 3.992ex; height: 2.661ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-61"></use><g transform="translate(500,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-64">\mathrm{a}_t^{l-1}</script>是第l-1层神经元的输出；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-65-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -874.2383890738297 1784.0109473217788 1099.4384584418224" style="width: 4.113ex; height: 2.54ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-66"></use><g transform="translate(568,362)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-65">f^{l-1}</script>是第l-1层的<strong>激活函数</strong>。</p><div class="md-section-divider"></div><p data-anchor-id="lheu"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-66-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2368.806049752705 44270.673387096766 4237.61209950541" style="width: 102.823ex; height: 9.798ex; vertical-align: -4.476ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(15592,0)"><g transform="translate(-15,0)"><g transform="translate(0,746)"><g transform="translate(120,0)"><rect stroke="none" width="3293" height="60" x="0" y="220"></rect><g transform="translate(489,757)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="1966" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></g><g transform="translate(60,-816)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></g></g><use href="#MJMAIN-3D" x="3811" y="0"></use></g><use href="#MJMAIN-3D" x="3811" y="-1563"></use></g><g transform="translate(4853,0)"><g transform="translate(0,746)"><g transform="translate(120,0)"><rect stroke="none" width="2403" height="60" x="0" y="220"></rect><g transform="translate(67,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="1966" y="513"></use></g></g><g transform="translate(60,-816)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-61"></use><g transform="translate(500,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-395"></use></g></g></g><g transform="translate(2930,0)"><rect stroke="none" width="3293" height="60" x="0" y="220"></rect><g transform="translate(505,757)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-61"></use><g transform="translate(500,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="707" y="-395"></use></g></g><g transform="translate(60,-816)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></g></g></g><g transform="translate(0,-1563)"><use href="#MJMATHI-55"></use><use href="#MJMATHI-64" x="767" y="0"></use><use href="#MJMATHI-69" x="1291" y="0"></use><use href="#MJMATHI-61" x="1636" y="0"></use><use href="#MJMATHI-67" x="2166" y="0"></use><use href="#MJMAIN-5B" x="2646" y="0"></use><g transform="translate(2925,0)"><use href="#MJMATHI-66"></use><g transform="translate(568,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g></g></g><use href="#MJMAIN-28" x="4903" y="0"></use><g transform="translate(5293,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g><use href="#MJMAIN-29" x="7899" y="0"></use><use href="#MJMAIN-5D" x="8288" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,746)"><g id="mjx-eqn-39"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1563)"><g id="mjx-eqn-40"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-66">
\begin{align}
\frac{\partial{\mathrm{net}_t^l}}{\partial{\mathrm{net}_t^{l-1}}}=&\frac{\partial{\mathrm{net}^l}}{\partial{\mathrm{a}_t^{l-1}}}\frac{\partial{\mathrm{a}_t^{l-1}}}{\partial{\mathrm{net}_t^{l-1}}}\\
=&Udiag[f'^{l-1}(\mathrm{net}_t^{l-1})]
\end{align}
</script><p></p><p data-anchor-id="3omo">所以，</p><div class="md-section-divider"></div><p data-anchor-id="88w6"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-67-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3805.3459431994647 44270.673387096766 7110.6918863989295" style="width: 102.823ex; height: 16.573ex; vertical-align: -7.742ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(12690,0)"><g transform="translate(-15,0)"><g transform="translate(0,2393)"><use href="#MJMAIN-28"></use><g transform="translate(389,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g><g transform="translate(2058,0)"><use href="#MJMAIN-29"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="550" y="583"></use></g><use href="#MJMAIN-3D" x="3324" y="0"></use></g><use href="#MJMAIN-3D" x="3324" y="-614"></use><use href="#MJMAIN-3D" x="3324" y="-2922"></use></g><g transform="translate(4366,0)"><g transform="translate(0,2393)"><g transform="translate(120,0)"><rect stroke="none" width="3293" height="60" x="0" y="220"></rect><g transform="translate(980,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-816)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></g></g></g><g transform="translate(0,-614)"><g transform="translate(120,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(550,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-816)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="1966" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></g></g><g transform="translate(2960,0)"><rect stroke="none" width="3293" height="60" x="0" y="220"></rect><g transform="translate(489,757)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="1966" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></g><g transform="translate(60,-816)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g></g></g></g><g transform="translate(0,-2922)"><use href="#MJMAIN-28"></use><g transform="translate(389,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="641" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g><g transform="translate(1189,0)"><use href="#MJMAIN-29"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="550" y="583"></use></g><use href="#MJMATHI-55" x="2177" y="0"></use><use href="#MJMATHI-64" x="2944" y="0"></use><use href="#MJMATHI-69" x="3468" y="0"></use><use href="#MJMATHI-61" x="3813" y="0"></use><use href="#MJMATHI-67" x="4343" y="0"></use><use href="#MJMAIN-5B" x="4823" y="0"></use><g transform="translate(5102,0)"><use href="#MJMATHI-66"></use><g transform="translate(568,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032"></use><g transform="translate(194,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g></g></g><use href="#MJMAIN-28" x="7081" y="0"></use><g transform="translate(7470,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,353)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="298" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1077" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-395"></use></g><use href="#MJMAIN-29" x="10076" y="0"></use><use href="#MJMAIN-5D" x="10466" y="0"></use><use href="#MJMAIN-28" x="12744" y="0"></use><g transform="translate(13134,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-34" x="13967" y="0"></use><use href="#MJMAIN-29" x="14467" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,2393)"><g id="mjx-eqn-41"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-614)"><g id="mjx-eqn-42"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2922)"><g id="mjx-eqn-43"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-67">
\begin{align}
(\delta_t^{l-1})^T=&\frac{\partial{E}}{\partial{\mathrm{net}_t^{l-1}}}\\
=&\frac{\partial{E}}{\partial{\mathrm{net}_t^l}}\frac{\partial{\mathrm{net}_t^l}}{\partial{\mathrm{net}_t^{l-1}}}\\
=&(\delta_t^l)^TUdiag[f'^{l-1}(\mathrm{net}_t^{l-1})]\qquad(式4)
\end{align}
</script><p></p><p data-anchor-id="mzvk"><strong>式4</strong>就是将误差项传递到上一层算法。</p><div class="md-section-divider"></div><h4 data-anchor-id="r2se" id="权重梯度的计算">权重梯度的计算</h4><p data-anchor-id="bipj">现在，我们终于来到了BPTT算法的最后一步：计算每个权重的梯度。</p><p data-anchor-id="2v3n">首先，我们计算误差函数E对权重矩阵W的梯度<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-68-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -949.4078778422705 1502.684558397461 1398.815755684541" style="width: 3.508ex; height: 3.266ex; vertical-align: -1.089ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="1262" height="60" x="0" y="220"></rect><g transform="translate(160,423)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="567" y="0"></use></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-68">\frac{\partial{E}}{\partial{W}}</script>。</p><p data-anchor-id="qzwz"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-f7d034c8f05812f7.png" alt="" title=""></p><p data-anchor-id="cb0s">上图展示了我们到目前为止，在前两步中已经计算得到的量，包括每个时刻t <strong>循环层</strong>的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-69-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 825.1191013989369 640.8854201102238" style="width: 1.935ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="663" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-69">s_t</script>，以及误差项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-70-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 800.1191013989369 914.8854201102238" style="width: 1.815ex; height: 2.177ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-70">\delta_t</script>。</p><p data-anchor-id="pkyc">回忆一下我们在文章<a href="https://www.zybuluo.com/hanbingtao/note/476663" target="_blank">零基础入门深度学习(3) - 神经网络和反向传播算法</a>介绍的全连接网络的权重梯度计算算法：只要知道了任意一个时刻的<strong>误差项</strong><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-71-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 800.1191013989369 914.8854201102238" style="width: 1.815ex; height: 2.177ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-71">\delta_t</script>，以及上一个时刻循环层的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-72-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -468.20006936799257 1654.5086745365313 645.8854201102238" style="width: 3.871ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-72">\mathrm{s}_{t-1}</script>，就可以按照下面的公式求出权重矩阵在t时刻的梯度<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-73-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 2695.020561473032 888.6635947032757" style="width: 6.29ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1048" y="0"></use></g><use href="#MJMATHI-45" x="1930" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-73">\nabla_{Wt}E</script>：</p><div class="md-section-divider"></div><p data-anchor-id="wi4t"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-74-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3414.457255839105 21983.521082310046 6328.91451167821" style="width: 51.048ex; height: 14.758ex; vertical-align: -6.895ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-74" x="1335" y="-213"></use></g><use href="#MJMATHI-45" x="1852" y="0"></use><use href="#MJMAIN-3D" x="2895" y="0"></use><g transform="translate(3951,0)"><g transform="translate(0,3395)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5646"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><use href="#MJMAIN-2E" x="1210" y="-162"></use><use href="#MJMAIN-2E" x="1210" y="-1362"></use><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g></g><g transform="translate(3684,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g></g><g transform="translate(7382,0)"><g transform="translate(0,2570)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1038)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2588)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(9551,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g></g><g transform="translate(13251,3395)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5646"></use></g></g><use href="#MJMAIN-28" x="19870" y="0"></use><g transform="translate(20260,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-35" x="21093" y="0"></use><use href="#MJMAIN-29" x="21594" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-74">
\nabla_{W_t}E=\begin{bmatrix}
\delta_1^ts_1^{t-1} & \delta_1^ts_2^{t-1} & ... &  \delta_1^ts_n^{t-1}\\
\delta_2^ts_1^{t-1} & \delta_2^ts_2^{t-1} & ... &  \delta_2^ts_n^{t-1}\\
.\\.\\
\delta_n^ts_1^{t-1} & \delta_n^ts_2^{t-1} & ... &  \delta_n^ts_n^{t-1}\\
\end{bmatrix}\qquad(式5)
</script><p></p><p data-anchor-id="2xlu">在<strong>式5</strong>中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-75-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -815.4845025461965 809.269101398937 1146.8313424339221" style="width: 1.935ex; height: 2.661ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="628" y="-430"></use></g></svg></span><script type="math/tex" id="MathJax-Element-75">\delta_i^t</script>表示t时刻<strong>误差项</strong>向量的第i个分量；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-76-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -843.7687737936584 1729.5086745365313 1175.115613681384" style="width: 3.992ex; height: 2.782ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="663" y="-430"></use></g></svg></span><script type="math/tex" id="MathJax-Element-76">s_i^{t-1}</script>表示t-1时刻<strong>循环层</strong>第i个神经元的输出值。</p><p data-anchor-id="txsf">我们下面可以简单推导一下<strong>式5</strong>。</p><p data-anchor-id="6dnw">我们知道：</p><div class="md-section-divider"></div><p data-anchor-id="dfle"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-77-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -7331.262377393911 44270.673387096766 14162.524754787823" style="width: 102.823ex; height: 32.903ex; vertical-align: -15.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(11272,0)"><g transform="translate(-15,0)"><g transform="translate(1861,6512)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use><use href="#MJMAIN-3D" x="2023" y="0"></use></g><g transform="translate(0,2619)"><g transform="translate(0,3302)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4348.880303800534) scale(1,4.30825921028281)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5460"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2502)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-435"></use></g></g><g transform="translate(35,997)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="511" y="-435"></use></g></g><use href="#MJMAIN-2E" x="837" y="-203"></use><use href="#MJMAIN-2E" x="837" y="-1403"></use><g transform="translate(0,-2603)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="511" y="-212"></use></g></g></g></g><g transform="translate(2940,3302)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4348.880303800534) scale(1,4.30825921028281)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5460"></use></g><use href="#MJMAIN-3D" x="3885" y="0"></use></g><use href="#MJMAIN-3D" x="3885" y="-3917"></use></g><g transform="translate(4927,0)"><g transform="translate(0,6512)"><use href="#MJMATHI-55"></use><g transform="translate(767,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="1873" y="0"></use><use href="#MJMATHI-57" x="2874" y="0"></use><g transform="translate(3923,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(0,2619)"><use href="#MJMATHI-55"></use><g transform="translate(767,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="1873" y="0"></use><g transform="translate(2874,0)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2350)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(35,1150)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><use href="#MJMAIN-2E" x="658" y="-50"></use><use href="#MJMAIN-2E" x="658" y="-1250"></use><g transform="translate(0,-2450)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g></g></g><g transform="translate(2580,0)"><g transform="translate(35,2350)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(35,1150)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(0,-2450)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g></g><g transform="translate(5175,0)"><g transform="translate(0,2350)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1150)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2450)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(7344,0)"><g transform="translate(35,2350)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g><g transform="translate(35,1150)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g><g transform="translate(0,-2450)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="600" y="0"></use></g></g></g></g><g transform="translate(10011,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g><g transform="translate(13553,0)"><g transform="translate(0,3341)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4427.172146680622) scale(1,4.438096428989423)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5538"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2516)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><g transform="translate(0,984)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-2E" x="725" y="-216"></use><use href="#MJMAIN-2E" x="725" y="-1416"></use><g transform="translate(0,-2642)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g><g transform="translate(2716,3341)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4427.172146680622) scale(1,4.438096428989423)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5538"></use></g></g></g><g transform="translate(0,-3917)"><use href="#MJMATHI-55"></use><g transform="translate(767,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="1873" y="0"></use><g transform="translate(2874,0)"><g transform="translate(0,3395)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5646"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(106,2570)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><g transform="translate(1524,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><use href="#MJMAIN-2B" x="3476" y="0"></use><g transform="translate(4476,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(6001,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-2E" x="7730" y="0"></use><use href="#MJMAIN-2E" x="8175" y="0"></use><use href="#MJMAIN-2E" x="8620" y="0"></use><g transform="translate(9066,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g><g transform="translate(10661,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(106,1038)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><g transform="translate(1524,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><use href="#MJMAIN-2B" x="3476" y="0"></use><g transform="translate(4476,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(6001,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-2E" x="7730" y="0"></use><use href="#MJMAIN-2E" x="8175" y="0"></use><use href="#MJMAIN-2E" x="8620" y="0"></use><g transform="translate(9066,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g><g transform="translate(10661,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><use href="#MJMAIN-2E" x="6162" y="-162"></use><use href="#MJMAIN-2E" x="6162" y="-1362"></use><g transform="translate(0,-2588)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g><g transform="translate(1595,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><use href="#MJMAIN-2B" x="3546" y="0"></use><g transform="translate(4547,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g><g transform="translate(6142,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-2E" x="7872" y="0"></use><use href="#MJMAIN-2E" x="8317" y="0"></use><use href="#MJMAIN-2E" x="8762" y="0"></use><g transform="translate(9207,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="600" y="0"></use></g></g><g transform="translate(10873,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g></g><g transform="translate(13589,3395)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5646"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,6512)"><g id="mjx-eqn-44"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,2619)"><g id="mjx-eqn-45"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-3917)"><g id="mjx-eqn-46"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-77">
\begin{align}
\mathrm{net}_t=&U\mathrm{x}_t+W\mathrm{s}_{t-1}\\
\begin{bmatrix}
net_1^t\\
net_2^t\\
.\\.\\
net_n^t\\
\end{bmatrix}=&U\mathrm{x}_t+
\begin{bmatrix}
w_{11} & w_{12} & ... & w_{1n}\\
w_{21} & w_{22} & ... & w_{2n}\\
.\\.\\
w_{n1} & w_{n2} & ... & w_{nn}\\
\end{bmatrix}
\begin{bmatrix}
s_1^{t-1}\\
s_2^{t-1}\\
.\\.\\
s_n^{t-1}\\
\end{bmatrix}\\
=&U\mathrm{x}_t+
\begin{bmatrix}
w_{11}s_1^{t-1}+w_{12}s_2^{t-1}...w_{1n}s_n^{t-1}\\
w_{21}s_1^{t-1}+w_{22}s_2^{t-1}...w_{2n}s_n^{t-1}\\
.\\.\\
w_{n1}s_1^{t-1}+w_{n2}s_2^{t-1}...w_{nn}s_n^{t-1}\\
\end{bmatrix}\\
\end{align}
</script><p></p><p data-anchor-id="x0nv">因为对W求导与<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-78-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1651.619101398937 880.8854201102238" style="width: 3.871ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-55"></use><g transform="translate(767,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-78">U\mathrm{x}_t</script>无关，我们不再考虑。现在，我们考虑对权重项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-79-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-79">w_{ji}</script>求导。通过观察上式我们可以看到<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-80-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-80">w_{ji}</script>只与<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-81-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -815.4845025461965 1820.181547239451 1283.302951202926" style="width: 4.234ex; height: 3.024ex; vertical-align: -1.21ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-430"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-81">net_j^t</script>有关，所以：</p><div class="md-section-divider"></div><p data-anchor-id="9cvp"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-82-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 10.403ex; vertical-align: -4.597ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2520.90428659536 44270.673387096766 4541.80857319072" style="width: 102.823ex; height: 10.524ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17671,0)"><g transform="translate(-15,0)"><g transform="translate(0,786)"><g transform="translate(120,0)"><rect stroke="none" width="2039" height="60" x="0" y="220"></rect><g transform="translate(353,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g><use href="#MJMAIN-3D" x="2557" y="0"></use></g><use href="#MJMAIN-3D" x="2557" y="-1554"></use></g><g transform="translate(3599,0)"><g transform="translate(0,786)"><g transform="translate(120,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(587,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-767)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-430"></use></g></g></g></g><g transform="translate(3034,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(60,918)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="511" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-430"></use></g></g></g><g transform="translate(293,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g><g transform="translate(0,-1554)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-430"></use><g transform="translate(836,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="663" y="-430"></use></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,786)"><g id="mjx-eqn-47"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1554)"><g id="mjx-eqn-48"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-82">
\begin{align}
\frac{\partial{E}}{\partial{w_{ji}}}=&\frac{\partial{E}}{\partial{net_j^t}}\frac{\partial{net_j^t}}{\partial{w_{ji}}}\\
=&\delta_j^ts_i^{t-1}
\end{align}
</script><p></p><p data-anchor-id="s41y">按照上面的规律就可以生成<strong>式5</strong>里面的矩阵。</p><p data-anchor-id="uxjb">我们已经求得了权重矩阵W在t时刻的梯度<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-83-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 2695.020561473032 888.6635947032757" style="width: 6.29ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1048" y="0"></use></g><use href="#MJMATHI-45" x="1930" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-83">\nabla_{Wt}E</script>，最终的梯度<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-84-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 2439.401460074095 888.6635947032757" style="width: 5.685ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="1178" y="-213"></use><use href="#MJMATHI-45" x="1674" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-84">\nabla_WE</script>是各个时刻的梯度<strong>之和</strong>：</p><div class="md-section-divider"></div><p data-anchor-id="gx2x"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-85-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -5022.1982458434995 44270.673387096766 9544.396491686999" style="width: 102.823ex; height: 22.137ex; vertical-align: -10.645ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(4174,0)"><g transform="translate(-15,0)"><g transform="translate(0,3308)"><use href="#MJMAIN-2207"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="1178" y="-213"></use><use href="#MJMATHI-45" x="1674" y="0"></use><use href="#MJMAIN-3D" x="2717" y="0"></use></g><use href="#MJMAIN-3D" x="2717" y="-1597"></use></g><g transform="translate(3759,0)"><g transform="translate(0,3308)"><use href="#MJSZ2-2211"></use><g transform="translate(147,-1090)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-69"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-3D" x="345" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1124" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="840" y="1627"></use><g transform="translate(1611,0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="1335" y="-213"></use></g></g><use href="#MJMATHI-45" x="3455" y="0"></use></g><g transform="translate(0,-1597)"><g transform="translate(0,3395)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5646"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><use href="#MJMAIN-2E" x="1210" y="-162"></use><use href="#MJMAIN-2E" x="1210" y="-1362"></use><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g></g><g transform="translate(3684,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g></g><g transform="translate(7382,0)"><g transform="translate(0,2570)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1038)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2588)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(9551,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g></g><g transform="translate(13251,3395)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5646"></use></g><use href="#MJMAIN-2B" x="13919" y="0"></use><use href="#MJMAIN-2E" x="14697" y="0"></use><use href="#MJMAIN-2E" x="15142" y="0"></use><use href="#MJMAIN-2E" x="15588" y="0"></use><use href="#MJMAIN-2B" x="16033" y="0"></use><g transform="translate(16811,0)"><g transform="translate(0,3406)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4558.1935177533) scale(1,4.655378968081758)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5669"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(30,2574)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(907,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><g transform="translate(30,1034)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(907,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><use href="#MJMAIN-2E" x="807" y="-166"></use><use href="#MJMAIN-2E" x="807" y="-1366"></use><g transform="translate(0,-2600)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g></g><g transform="translate(2878,0)"><g transform="translate(30,2574)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(907,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(30,1034)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(907,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(0,-2600)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g></g><g transform="translate(5770,0)"><g transform="translate(0,2574)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1034)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2600)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(7939,0)"><g transform="translate(30,2574)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(907,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(30,1034)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(907,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(0,-2600)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="663" y="510"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g></g><g transform="translate(10904,3406)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4558.1935177533) scale(1,4.655378968081758)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5669"></use></g></g><use href="#MJMAIN-28" x="30383" y="0"></use><g transform="translate(30773,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-36" x="31606" y="0"></use><use href="#MJMAIN-29" x="32106" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,3308)"><g id="mjx-eqn-49"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1597)"><g id="mjx-eqn-50"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-85">
\begin{align}
\nabla_WE=&\sum_{i=1}^t\nabla_{W_i}E\\
=&\begin{bmatrix}
\delta_1^ts_1^{t-1} & \delta_1^ts_2^{t-1} & ... &  \delta_1^ts_n^{t-1}\\
\delta_2^ts_1^{t-1} & \delta_2^ts_2^{t-1} & ... &  \delta_2^ts_n^{t-1}\\
.\\.\\
\delta_n^ts_1^{t-1} & \delta_n^ts_2^{t-1} & ... &  \delta_n^ts_n^{t-1}\\
\end{bmatrix}
+...+
\begin{bmatrix}
\delta_1^1s_1^0 & \delta_1^1s_2^0 & ... &  \delta_1^1s_n^0\\
\delta_2^1s_1^0 & \delta_2^1s_2^0 & ... &  \delta_2^1s_n^0\\
.\\.\\
\delta_n^1s_1^0 & \delta_n^1s_2^0 & ... &  \delta_n^1s_n^0\\
\end{bmatrix}\qquad(式6)
\end{align}
</script><p></p><p data-anchor-id="6fv6"><strong>式6</strong>就是计算<strong>循环层</strong>权重矩阵W的梯度的公式。</p><p data-anchor-id="wmd2"><code>----------数学公式超高能预警----------</code></p><p data-anchor-id="wrwa">前面已经介绍了<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-86-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 2439.401460074095 888.6635947032757" style="width: 5.685ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="1178" y="-213"></use><use href="#MJMATHI-45" x="1674" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-86">\nabla_WE</script>的计算方法，看上去还是比较直观的。然而，读者也许会困惑，为什么最终的梯度是各个时刻的梯度<strong>之和</strong>呢？我们前面只是直接用了这个结论，实际上这里面是有道理的，只是这个数学推导比较绕脑子。感兴趣的同学可以仔细阅读接下来这一段，它用到了矩阵对矩阵求导、张量与向量相乘运算的一些法则。</p><p data-anchor-id="tkoo">我们还是从这个式子开始：</p><div class="md-section-divider"></div><p data-anchor-id="3mbn"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-87-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 10983.246877334406 1040.4001387359851" style="width: 25.524ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use><use href="#MJMAIN-3D" x="2023" y="0"></use><use href="#MJMATHI-55" x="3080" y="0"></use><g transform="translate(3847,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g><use href="#MJMAIN-2B" x="4954" y="0"></use><use href="#MJMATHI-57" x="5954" y="0"></use><use href="#MJMATHI-66" x="7003" y="0"></use><use href="#MJMAIN-28" x="7553" y="0"></use><g transform="translate(7943,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="10593" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-87">
\mathrm{net}_t=U\mathrm{x}_t+Wf(\mathrm{net}_{t-1})
</script><p></p><p data-anchor-id="zdj9">因为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-88-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1651.619101398937 880.8854201102238" style="width: 3.871ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-55"></use><g transform="translate(767,0)"><use href="#MJMAIN-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="747" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-88">U\mathrm{x}_t</script>与W完全无关，我们把它看做常量。现在，考虑第一个式子加号右边的部分，因为W和<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-89-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 3980.0086745365315 1040.4001387359851" style="width: 9.194ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="3590" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-89">f(\mathrm{net}_{t-1})</script>都是W的函数，因此我们要用到大学里面都学过的导数乘法运算：</p><div class="md-section-divider"></div><p data-anchor-id="sxwc"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-90-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -828.7789736136457 7394.423754650683 1098.9790429816383" style="width: 17.177ex; height: 2.54ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-28"></use><use href="#MJMATHI-75" x="389" y="0"></use><use href="#MJMATHI-76" x="962" y="0"></use><g transform="translate(1447,0)"><use href="#MJMAIN-29"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="550" y="583"></use></g><use href="#MJMAIN-3D" x="2409" y="0"></use><g transform="translate(3465,0)"><use href="#MJMATHI-75"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="809" y="583"></use></g><use href="#MJMATHI-76" x="4333" y="0"></use><use href="#MJMAIN-2B" x="5040" y="0"></use><use href="#MJMATHI-75" x="6041" y="0"></use><g transform="translate(6614,0)"><use href="#MJMATHI-76"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="686" y="583"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-90">
(uv)'=u'v+uv'
</script><p></p><p data-anchor-id="f50z">因此，上面第一个式子写成：</p><div class="md-section-divider"></div><p data-anchor-id="mn8a"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true"><span class="MathJax_SVG" id="MathJax-Element-91-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1491.4079322712182 44270.673387096766 2710.6080016392107" style="width: 102.823ex; height: 6.29ex; vertical-align: -2.903ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(13397,0)"><g transform="translate(120,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(408,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><use href="#MJMAIN-3D" x="2951" y="0"></use><g transform="translate(4127,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><use href="#MJMATHI-66" x="6150" y="0"></use><use href="#MJMAIN-28" x="6700" y="0"></use><g transform="translate(7090,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="9740" y="0"></use><use href="#MJMAIN-2B" x="10352" y="0"></use><use href="#MJMATHI-57" x="11353" y="0"></use><g transform="translate(12688,0)"><rect stroke="none" width="4667" height="60" x="0" y="220"></rect><g transform="translate(60,721)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="3590" y="0"></use></g></g><g transform="translate(1525,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-91">
\frac{\partial{\mathrm{net}_t}}{\partial{W}}=\frac{\partial{W}}{\partial{W}}f(\mathrm{net}_{t-1})+W\frac{\partial{f(\mathrm{net}_{t-1})}}{\partial{W}}\\
</script><p></p><p data-anchor-id="r4av">我们最终需要计算的是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-92-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 2439.401460074095 888.6635947032757" style="width: 5.685ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="1178" y="-213"></use><use href="#MJMATHI-45" x="1674" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-92">\nabla_WE</script>：</p><div class="md-section-divider"></div><p data-anchor-id="z2sb"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-93-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 16.573ex; vertical-align: -7.742ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3829.366435861563 44270.673387096766 7158.732871723126" style="width: 102.823ex; height: 16.573ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(10413,0)"><g transform="translate(-15,0)"><g transform="translate(0,2417)"><use href="#MJMAIN-2207"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="1178" y="-213"></use><use href="#MJMATHI-45" x="1674" y="0"></use><use href="#MJMAIN-3D" x="2717" y="0"></use></g><use href="#MJMAIN-3D" x="2717" y="15"></use><use href="#MJMAIN-3D" x="2717" y="-2601"></use></g><g transform="translate(3759,0)"><g transform="translate(0,2417)"><g transform="translate(120,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(202,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,15)"><g transform="translate(120,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(550,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g></g><g transform="translate(2960,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(408,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,-2601)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><g transform="translate(1338,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><use href="#MJMATHI-66" x="3361" y="0"></use><use href="#MJMAIN-28" x="3911" y="0"></use><g transform="translate(4301,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="6951" y="0"></use><use href="#MJMAIN-2B" x="7563" y="0"></use><g transform="translate(8564,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g><use href="#MJMATHI-57" x="9615" y="0"></use><g transform="translate(10951,0)"><rect stroke="none" width="4667" height="60" x="0" y="220"></rect><g transform="translate(60,721)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="3590" y="0"></use></g></g><g transform="translate(1525,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><use href="#MJMAIN-28" x="17905" y="0"></use><g transform="translate(18294,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-37" x="19128" y="0"></use><use href="#MJMAIN-29" x="19628" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,2417)"><g id="mjx-eqn-51"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,15)"><g id="mjx-eqn-52"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2601)"><g id="mjx-eqn-53"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-93">
\begin{align}
\nabla_WE=&\frac{\partial{E}}{\partial{W}}\\
=&\frac{\partial{E}}{\partial{\mathrm{net}_t}}\frac{\partial{\mathrm{net}_t}}{\partial{W}}\\
=&\delta_t^T\frac{\partial{W}}{\partial{W}}f(\mathrm{net}_{t-1})+ \delta_t^TW\frac{\partial{f(\mathrm{net}_{t-1})}}{\partial{W}}\qquad(式7)\\
\end{align}
</script><p></p><p data-anchor-id="mpr4">我们先计算<strong>式7</strong>加号左边的部分。<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-94-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -949.4078778422705 1502.684558397461 1398.815755684541" style="width: 3.508ex; height: 3.266ex; vertical-align: -1.089ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="1262" height="60" x="0" y="220"></rect><g transform="translate(60,423)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="567" y="0"></use></g><g transform="translate(60,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="567" y="0"></use></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-94">\frac{\partial{W}}{\partial{W}}</script>是<strong>矩阵对矩阵求导</strong>，其结果是一个四维<strong>张量(tensor)</strong>，如下所示：</p><div class="md-section-divider"></div><p data-anchor-id="qgmg"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-95-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -14420.575580139028 44270.673387096766 28341.151160278056" style="width: 102.823ex; height: 65.806ex; vertical-align: -32.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(4922,0)"><g transform="translate(-15,0)"><g transform="translate(0,9737)"><g transform="translate(120,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><use href="#MJMAIN-3D" x="2253" y="0"></use></g><use href="#MJMAIN-3D" x="2253" y="-554"></use><use href="#MJMAIN-3D" x="2253" y="-10291"></use></g><g transform="translate(3295,0)"><g transform="translate(0,9737)"><g transform="translate(0,4663)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-7067.331119191946) scale(1,8.816469517731253)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-8183"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,3271)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(297,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(35,769)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(297,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><use href="#MJMAIN-2E" x="1122" y="-461"></use><use href="#MJMAIN-2E" x="1122" y="-1661"></use><g transform="translate(0,-3455)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g></g></g><g transform="translate(333,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g></g><g transform="translate(3508,0)"><g transform="translate(35,3271)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(297,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(35,769)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(297,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,-3455)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g></g><g transform="translate(333,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g></g><g transform="translate(7030,0)"><g transform="translate(0,3271)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,769)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-3455)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(9199,0)"><g transform="translate(35,3271)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g><g transform="translate(333,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(35,769)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g><g transform="translate(333,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,-3455)"><g transform="translate(120,0)"><rect stroke="none" width="2353" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="600" y="0"></use></g></g></g><g transform="translate(368,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g></g></g><g transform="translate(12794,4663)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-7067.331119191946) scale(1,8.816469517731253)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-8183"></use></g></g><g transform="translate(0,-554)"><g transform="translate(0,5827)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-9390.933559599227) scale(1,12.669873233166214)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-10510"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,960)"><g transform="translate(0,4866)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-7473.114023175386) scale(1,9.489409657007274)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-8589"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,3474)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g></g><g transform="translate(35,837)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g></g><use href="#MJMAIN-2E" x="1122" y="-529"></use><use href="#MJMAIN-2E" x="1122" y="-1729"></use><g transform="translate(0,-3523)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(95,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g></g></g></g></g></g><g transform="translate(3508,0)"><g transform="translate(35,3474)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g></g><g transform="translate(35,837)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g></g><g transform="translate(0,-3523)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(95,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g></g></g></g></g><g transform="translate(7030,0)"><g transform="translate(0,3474)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,837)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-3523)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(9199,0)"><g transform="translate(0,3474)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(382,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><g transform="translate(0,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g></g><g transform="translate(0,837)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(382,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><g transform="translate(0,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g></g><g transform="translate(0,-3523)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g><g transform="translate(347,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><g transform="translate(0,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="600" y="0"></use></g></g></g></g></g></g></g><g transform="translate(12652,4866)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-7473.114023175386) scale(1,9.489409657007274)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-8589"></use></g></g><use href="#MJMAIN-2E" x="6520" y="-3928"></use><use href="#MJMAIN-2E" x="6520" y="-5128"></use></g><g transform="translate(14305,0)"><g transform="translate(0,960)"><g transform="translate(0,4866)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-7473.114023175386) scale(1,9.489409657007274)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-8589"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,3474)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g></g><g transform="translate(35,837)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g></g><use href="#MJMAIN-2E" x="1122" y="-529"></use><use href="#MJMAIN-2E" x="1122" y="-1729"></use><g transform="translate(0,-3523)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(95,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g></g></g></g></g></g><g transform="translate(3508,0)"><g transform="translate(35,3474)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g></g><g transform="translate(35,837)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g></g><g transform="translate(0,-3523)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(95,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g></g></g></g></g><g transform="translate(7030,0)"><g transform="translate(0,3474)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,837)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-3523)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(9199,0)"><g transform="translate(0,3474)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(382,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><g transform="translate(0,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g></g><g transform="translate(0,837)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(382,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><g transform="translate(0,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g></g><g transform="translate(0,-3523)"><g transform="translate(120,0)"><rect stroke="none" width="2211" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g><g transform="translate(347,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><g transform="translate(0,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="600" y="0"></use></g></g></g></g></g></g></g><g transform="translate(12652,4866)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-7473.114023175386) scale(1,9.489409657007274)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-8589"></use></g></g></g><g transform="translate(28626,0)"><g transform="translate(0,960)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g></g><g transform="translate(30796,5827)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-9390.933559599227) scale(1,12.669873233166214)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-10510"></use></g></g><g transform="translate(0,-10291)"><g transform="translate(0,4110)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-5963.014900662252) scale(1,6.985099337748345)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-7076"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,960)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><use href="#MJMAIN-31" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-2E" x="111" y="-50"></use><use href="#MJMAIN-2E" x="111" y="-1250"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g><g transform="translate(1486,0)"><use href="#MJMAIN-30" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g><g transform="translate(2986,0)"><g transform="translate(0,2350)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1150)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2450)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(5155,0)"><use href="#MJMAIN-30" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g></g><g transform="translate(6657,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g><use href="#MJMAIN-2E" x="3523" y="-2211"></use><use href="#MJMAIN-2E" x="3523" y="-3411"></use></g><g transform="translate(8310,0)"><g transform="translate(0,960)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><use href="#MJMAIN-30" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-2E" x="111" y="-50"></use><use href="#MJMAIN-2E" x="111" y="-1250"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g><g transform="translate(1486,0)"><use href="#MJMAIN-31" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g><g transform="translate(2986,0)"><g transform="translate(0,2350)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1150)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2450)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(5155,0)"><use href="#MJMAIN-30" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g></g><g transform="translate(6657,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g></g><g transform="translate(16635,0)"><g transform="translate(0,960)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g></g><g transform="translate(18805,4110)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-5963.014900662252) scale(1,6.985099337748345)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-7076"></use></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,9737)"><g id="mjx-eqn-54"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-554)"><g id="mjx-eqn-55"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-10291)"><g id="mjx-eqn-56"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-95">
\begin{align}
\frac{\partial{W}}{\partial{W}}=&
\begin{bmatrix}
\frac{\partial{w_{11}}}{\partial{W}} & \frac{\partial{w_{12}}}{\partial{W}} & ... & \frac{\partial{w_{1n}}}{\partial{W}}\\
\frac{\partial{w_{21}}}{\partial{W}} & \frac{\partial{w_{22}}}{\partial{W}} & ... & \frac{\partial{w_{2n}}}{\partial{W}}\\
.\\.\\
\frac{\partial{w_{n1}}}{\partial{W}} & \frac{\partial{w_{n2}}}{\partial{W}} & ... & \frac{\partial{w_{nn}}}{\partial{W}}\\
\end{bmatrix}\\
=&
\begin{bmatrix}
\begin{bmatrix}
\frac{\partial{w_{11}}}{\partial{w_{11}}} & \frac{\partial{w_{11}}}{\partial{w_{12}}} & ... & \frac{\partial{w_{11}}}{\partial{_{1n}}}\\
\frac{\partial{w_{11}}}{\partial{w_{21}}} & \frac{\partial{w_{11}}}{\partial{w_{22}}} & ... & \frac{\partial{w_{11}}}{\partial{_{2n}}}\\
.\\.\\
\frac{\partial{w_{11}}}{\partial{w_{n1}}} & \frac{\partial{w_{11}}}{\partial{w_{n2}}} & ... & \frac{\partial{w_{11}}}{\partial{_{nn}}}\\
\end{bmatrix} &
\begin{bmatrix}
\frac{\partial{w_{12}}}{\partial{w_{11}}} & \frac{\partial{w_{12}}}{\partial{w_{12}}} & ... & \frac{\partial{w_{12}}}{\partial{_{1n}}}\\
\frac{\partial{w_{12}}}{\partial{w_{21}}} & \frac{\partial{w_{12}}}{\partial{w_{22}}} & ... & \frac{\partial{w_{12}}}{\partial{_{2n}}}\\
.\\.\\
\frac{\partial{w_{12}}}{\partial{w_{n1}}} & \frac{\partial{w_{12}}}{\partial{w_{n2}}} & ... & \frac{\partial{w_{12}}}{\partial{_{nn}}}\\
\end{bmatrix}&...\\
.\\.\\
\end{bmatrix}\\
=&
\begin{bmatrix}
\begin{bmatrix}
1 & 0 & ... & 0\\
0 & 0 & ... & 0\\
.\\.\\
0 & 0 & ... & 0\\
\end{bmatrix} &
\begin{bmatrix}
0 & 1 & ... & 0\\
0 & 0 & ... & 0\\
.\\.\\
0 & 0 & ... & 0\\
\end{bmatrix}&...\\
.\\.\\
\end{bmatrix}\\
\end{align}
</script><p></p><p data-anchor-id="3l9c">接下来，我们知道<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-96-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 7043.572904628618 1040.4001387359851" style="width: 16.331ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><g transform="translate(469,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use href="#MJMAIN-3D" x="2007" y="0"></use><use href="#MJMATHI-66" x="3063" y="0"></use><use href="#MJMAIN-28" x="3614" y="0"></use><g transform="translate(4003,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="6654" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-96">s_{t-1}=f({\mathrm{net}_{t-1}})</script>，它是一个<strong>列向量</strong>。我们让上面的四维张量与这个向量相乘，得到了一个三维张量，再左乘行向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-97-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -843.7687737936584 1051.8067273459228 1150.3668763398548" style="width: 2.419ex; height: 2.661ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g></svg></span><script type="math/tex" id="MathJax-Element-97">\delta_t^T</script>，最终得到一个矩阵：</p><div class="md-section-divider"></div><p data-anchor-id="4qof"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-98-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -17497.288021548997 44270.673387096766 34494.57604309799" style="width: 102.823ex; height: 80.081ex; vertical-align: -39.556ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(6017,0)"><g transform="translate(-15,0)"><g transform="translate(0,16085)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><g transform="translate(1338,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><use href="#MJMATHI-66" x="3361" y="0"></use><use href="#MJMAIN-28" x="3911" y="0"></use><g transform="translate(4301,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="6951" y="0"></use><use href="#MJMAIN-3D" x="7618" y="0"></use></g><use href="#MJMAIN-3D" x="7618" y="10965"></use><use href="#MJMAIN-3D" x="7618" y="2845"></use><use href="#MJMAIN-3D" x="7618" y="-5376"></use><use href="#MJMAIN-3D" x="7618" y="-12781"></use><use href="#MJMAIN-3D" x="7618" y="-16728"></use></g><g transform="translate(8661,0)"><g transform="translate(0,16085)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><g transform="translate(1338,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><g transform="translate(3361,0)"><use href="#MJMAIN-73"></use><g transform="translate(394,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(0,10965)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><g transform="translate(1051,0)"><g transform="translate(0,4110)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-5963.014900662252) scale(1,6.985099337748345)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-7076"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,960)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><use href="#MJMAIN-31" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-2E" x="111" y="-50"></use><use href="#MJMAIN-2E" x="111" y="-1250"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g><g transform="translate(1486,0)"><use href="#MJMAIN-30" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g><g transform="translate(2986,0)"><g transform="translate(0,2350)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1150)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2450)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(5155,0)"><use href="#MJMAIN-30" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g></g><g transform="translate(6657,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g><use href="#MJMAIN-2E" x="3523" y="-2211"></use><use href="#MJMAIN-2E" x="3523" y="-3411"></use></g><g transform="translate(8310,0)"><g transform="translate(0,960)"><g transform="translate(0,3150)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5155"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><use href="#MJMAIN-30" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-2E" x="111" y="-50"></use><use href="#MJMAIN-2E" x="111" y="-1250"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g><g transform="translate(1486,0)"><use href="#MJMAIN-31" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g><g transform="translate(2986,0)"><g transform="translate(0,2350)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1150)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2450)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(5155,0)"><use href="#MJMAIN-30" x="0" y="2350"></use><use href="#MJMAIN-30" x="0" y="1150"></use><use href="#MJMAIN-30" x="0" y="-2450"></use></g></g><g transform="translate(6657,3150)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4045.1953642384105) scale(1,3.8046357615894038)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5155"></use></g></g></g><g transform="translate(16635,0)"><g transform="translate(0,960)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g></g><g transform="translate(18805,4110)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-5963.014900662252) scale(1,6.985099337748345)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-7076"></use></g></g><g transform="translate(20525,0)"><g transform="translate(0,3341)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4427.172146680622) scale(1,4.438096428989423)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5538"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2516)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><g transform="translate(0,984)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-2E" x="725" y="-216"></use><use href="#MJMAIN-2E" x="725" y="-1416"></use><g transform="translate(0,-2642)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g><g transform="translate(2716,3341)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4427.172146680622) scale(1,4.438096428989423)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5538"></use></g></g></g><g transform="translate(0,2845)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><g transform="translate(1051,0)"><g transform="translate(0,4210)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-6162.15748188368) scale(1,7.3153523745998)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-7276"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,960)"><g transform="translate(0,3249)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4244.337945459839) scale(1,4.13488879844086)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5355"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2425)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><use href="#MJMAIN-30" x="614" y="1050"></use><use href="#MJMAIN-2E" x="725" y="-150"></use><use href="#MJMAIN-2E" x="725" y="-1350"></use><use href="#MJMAIN-30" x="614" y="-2550"></use></g></g><g transform="translate(2716,3249)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4244.337945459839) scale(1,4.13488879844086)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5355"></use></g></g><use href="#MJMAIN-2E" x="1552" y="-2311"></use><use href="#MJMAIN-2E" x="1552" y="-3511"></use></g><g transform="translate(4369,0)"><g transform="translate(0,960)"><g transform="translate(0,3249)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4244.337945459839) scale(1,4.13488879844086)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5355"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2425)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-30" x="614" y="1050"></use><use href="#MJMAIN-2E" x="725" y="-150"></use><use href="#MJMAIN-2E" x="725" y="-1350"></use><use href="#MJMAIN-30" x="614" y="-2550"></use></g></g><g transform="translate(2716,3249)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4244.337945459839) scale(1,4.13488879844086)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5355"></use></g></g></g><g transform="translate(8753,0)"><g transform="translate(0,960)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g></g><g transform="translate(10924,4210)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-6162.15748188368) scale(1,7.3153523745998)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-7276"></use></g></g></g><g transform="translate(0,-5376)"><use href="#MJSZ1-5B"></use><g transform="translate(584,0)"><g transform="translate(-15,0)"><g transform="translate(0,3)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use></g></g><g transform="translate(1884,0)"><g transform="translate(0,3)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use></g></g><g transform="translate(3782,0)"><g transform="translate(0,3)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(5951,0)"><g transform="translate(0,3)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use></g></g></g><use href="#MJSZ1-5D" x="7671" y="-1"></use><g transform="translate(8089,0)"><g transform="translate(0,4210)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-6162.15748188368) scale(1,7.3153523745998)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-7276"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,960)"><g transform="translate(0,3249)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4244.337945459839) scale(1,4.13488879844086)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5355"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2425)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g><use href="#MJMAIN-30" x="614" y="1050"></use><use href="#MJMAIN-2E" x="725" y="-150"></use><use href="#MJMAIN-2E" x="725" y="-1350"></use><use href="#MJMAIN-30" x="614" y="-2550"></use></g></g><g transform="translate(2716,3249)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4244.337945459839) scale(1,4.13488879844086)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5355"></use></g></g><use href="#MJMAIN-2E" x="1552" y="-2311"></use><use href="#MJMAIN-2E" x="1552" y="-3511"></use></g><g transform="translate(4369,0)"><g transform="translate(0,960)"><g transform="translate(0,3249)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4244.337945459839) scale(1,4.13488879844086)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5355"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2425)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g><use href="#MJMAIN-30" x="614" y="1050"></use><use href="#MJMAIN-2E" x="725" y="-150"></use><use href="#MJMAIN-2E" x="725" y="-1350"></use><use href="#MJMAIN-30" x="614" y="-2550"></use></g></g><g transform="translate(2716,3249)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4244.337945459839) scale(1,4.13488879844086)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5355"></use></g></g></g><g transform="translate(8753,0)"><g transform="translate(0,960)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g></g><g transform="translate(10924,4210)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-6162.15748188368) scale(1,7.3153523745998)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-7276"></use></g></g></g><g transform="translate(0,-12781)"><g transform="translate(0,3395)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5646"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g><use href="#MJMAIN-2E" x="1210" y="-162"></use><use href="#MJMAIN-2E" x="1210" y="-1362"></use><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="663" y="-435"></use></g></g></g><g transform="translate(3684,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="663" y="-435"></use></g></g></g><g transform="translate(7382,0)"><g transform="translate(0,2570)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1038)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2588)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(9551,0)"><g transform="translate(35,2570)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(35,1038)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g><g transform="translate(0,-2588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-73"></use><g transform="translate(469,352)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-212"></use></g></g></g></g><g transform="translate(13251,3395)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4534.897627291659) scale(1,4.616745650566598)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5646"></use></g></g><g transform="translate(0,-16728)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1048" y="0"></use></g><use href="#MJMATHI-45" x="1930" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,16085)"><g id="mjx-eqn-57"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,10965)"><g id="mjx-eqn-58"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,2845)"><g id="mjx-eqn-59"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-5376)"><g id="mjx-eqn-60"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-12781)"><g id="mjx-eqn-61"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-16728)"><g id="mjx-eqn-62"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-98">
\begin{align}
\delta_t^T\frac{\partial{W}}{\partial{W}}f({\mathrm{net}_{t-1}})=&
\delta_t^T\frac{\partial{W}}{\partial{W}}{\mathrm{s}_{t-1}}\\
=&\delta_t^T
\begin{bmatrix}
\begin{bmatrix}
1 & 0 & ... & 0\\
0 & 0 & ... & 0\\
.\\.\\
0 & 0 & ... & 0\\
\end{bmatrix} &
\begin{bmatrix}
0 & 1 & ... & 0\\
0 & 0 & ... & 0\\
.\\.\\
0 & 0 & ... & 0\\
\end{bmatrix}&...\\
.\\.\\
\end{bmatrix}
\begin{bmatrix}
s_1^{t-1}\\
s_2^{t-1}\\
.\\.\\
s_n^{t-1}\\
\end{bmatrix}\\
=&\delta_t^T
\begin{bmatrix}
\begin{bmatrix}
s_1^{t-1}\\
0\\
.\\.\\
0\\
\end{bmatrix} &
\begin{bmatrix}
s_2^{t-1}\\
0\\
.\\.\\
0\\
\end{bmatrix}&...\\
.\\.\\
\end{bmatrix}\\
=&
\begin{bmatrix}
\delta_1^t & \delta_2^t & ... &\delta_n^t
\end{bmatrix}
\begin{bmatrix}
\begin{bmatrix}
s_1^{t-1}\\
0\\
.\\.\\
0\\
\end{bmatrix} &
\begin{bmatrix}
s_2^{t-1}\\
0\\
.\\.\\
0\\
\end{bmatrix}&...\\
.\\.\\
\end{bmatrix}\\
=&
\begin{bmatrix}
\delta_1^ts_1^{t-1} & \delta_1^ts_2^{t-1} & ... &  \delta_1^ts_n^{t-1}\\
\delta_2^ts_1^{t-1} & \delta_2^ts_2^{t-1} & ... &  \delta_2^ts_n^{t-1}\\
.\\.\\
\delta_n^ts_1^{t-1} & \delta_n^ts_2^{t-1} & ... &  \delta_n^ts_n^{t-1}\\
\end{bmatrix}\\
=&\nabla_{Wt}E
\end{align}
</script><p></p><p data-anchor-id="22da">接下来，我们计算<strong>式7</strong>加号右边的部分：</p><div class="md-section-divider"></div><p data-anchor-id="rqp0"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-99-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -5097.967008000296 44270.673387096766 9695.934016000592" style="width: 102.823ex; height: 22.5ex; vertical-align: -10.766ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(12595,0)"><g transform="translate(-15,0)"><g transform="translate(0,3606)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><use href="#MJMATHI-57" x="1051" y="0"></use><g transform="translate(2386,0)"><rect stroke="none" width="4667" height="60" x="0" y="220"></rect><g transform="translate(60,721)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="3590" y="0"></use></g></g><g transform="translate(1525,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><use href="#MJMAIN-3D" x="7452" y="0"></use></g><use href="#MJMAIN-3D" x="7452" y="1069"></use><use href="#MJMAIN-3D" x="7452" y="-1333"></use><use href="#MJMAIN-3D" x="7452" y="-3870"></use></g><g transform="translate(8494,0)"><g transform="translate(0,3606)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><use href="#MJMATHI-57" x="1051" y="0"></use><g transform="translate(2386,0)"><rect stroke="none" width="4667" height="60" x="0" y="220"></rect><g transform="translate(60,721)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="3590" y="0"></use></g></g><g transform="translate(724,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g><g transform="translate(7461,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(861,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,1069)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><use href="#MJMATHI-57" x="1051" y="0"></use><g transform="translate(2100,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="2963" y="0"></use><g transform="translate(3353,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-29" x="6003" y="0"></use><g transform="translate(6679,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(861,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,-1333)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><g transform="translate(1338,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(512,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g></g><g transform="translate(5083,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(861,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,-3870)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><g transform="translate(444,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g><g transform="translate(1991,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(861,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,3606)"><g id="mjx-eqn-63"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,1069)"><g id="mjx-eqn-64"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1333)"><g id="mjx-eqn-65"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-3870)"><g id="mjx-eqn-66"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-99">
\begin{align}
\delta_t^TW\frac{\partial{f(\mathrm{net}_{t-1})}}{\partial{W}}=&
\delta_t^TW\frac{\partial{f(\mathrm{net}_{t-1})}}{\partial{\mathrm{net}_{t-1}}}\frac{\partial{\mathrm{net}_{t-1}}}{\partial{W}}\\
=&\delta_t^TWf'(\mathrm{net}_{t-1})\frac{\partial{\mathrm{net}_{t-1}}}{\partial{W}}\\
=&\delta_t^T\frac{\partial{\mathrm{net}_t}}{\partial{\mathrm{net}_{t-1}}}\frac{\partial{\mathrm{net}_{t-1}}}{\partial{W}}\\
=&\delta_{t-1}^T\frac{\partial{\mathrm{net}_{t-1}}}{\partial{W}}\\
\end{align}
</script><p></p><p data-anchor-id="zlqw">于是，我们得到了如下递推公式：</p><div class="md-section-divider"></div><p data-anchor-id="mu52"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-100-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -7247.347658912991 44270.673387096766 13994.695317825983" style="width: 102.823ex; height: 32.54ex; vertical-align: -15.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(13327,0)"><g transform="translate(-15,0)"><g transform="translate(0,5835)"><use href="#MJMAIN-2207"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="1178" y="-213"></use><use href="#MJMATHI-45" x="1674" y="0"></use><use href="#MJMAIN-3D" x="2717" y="0"></use></g><use href="#MJMAIN-3D" x="2717" y="3433"></use><use href="#MJMAIN-3D" x="2717" y="896"></use><use href="#MJMAIN-3D" x="2717" y="-1505"></use><use href="#MJMAIN-3D" x="2717" y="-3266"></use><use href="#MJMAIN-3D" x="2717" y="-5511"></use></g><g transform="translate(3759,0)"><g transform="translate(0,5835)"><g transform="translate(120,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(202,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,3433)"><g transform="translate(120,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(550,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g></g><g transform="translate(2960,0)"><rect stroke="none" width="2433" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1966" y="-213"></use></g></g><g transform="translate(408,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,896)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1048" y="0"></use></g><use href="#MJMATHI-45" x="1930" y="0"></use><use href="#MJMAIN-2B" x="2917" y="0"></use><g transform="translate(3917,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><g transform="translate(444,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><g transform="translate(5909,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g></g><g transform="translate(861,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,-1505)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1048" y="0"></use></g><use href="#MJMATHI-45" x="1930" y="0"></use><use href="#MJMAIN-2B" x="2917" y="0"></use><g transform="translate(3917,0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1048" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="1410" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="2188" y="0"></use></g></g><use href="#MJMATHI-45" x="6752" y="0"></use><use href="#MJMAIN-2B" x="7739" y="0"></use><g transform="translate(8740,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><g transform="translate(444,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1140" y="0"></use></g></g><g transform="translate(10731,0)"><rect stroke="none" width="3338" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><g transform="translate(1390,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1140" y="0"></use></g></g></g><g transform="translate(861,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g><g transform="translate(0,-3266)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1048" y="0"></use></g><use href="#MJMATHI-45" x="1930" y="0"></use><use href="#MJMAIN-2B" x="2917" y="0"></use><g transform="translate(3917,0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1048" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="1410" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="2188" y="0"></use></g></g><use href="#MJMATHI-45" x="6752" y="0"></use><use href="#MJMAIN-2B" x="7517" y="0"></use><use href="#MJMAIN-2E" x="8295" y="0"></use><use href="#MJMAIN-2E" x="8741" y="0"></use><use href="#MJMAIN-2E" x="9186" y="0"></use><use href="#MJMAIN-2B" x="9631" y="0"></use><g transform="translate(10409,0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1048" y="0"></use></g></g><use href="#MJMATHI-45" x="12438" y="0"></use></g><g transform="translate(0,-5511)"><use href="#MJSZ2-2211"></use><g transform="translate(85,-1110)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-3D" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1300" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="840" y="1627"></use><g transform="translate(1611,0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1048" y="0"></use></g></g><use href="#MJMATHI-45" x="3654" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,5835)"><g id="mjx-eqn-67"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,3433)"><g id="mjx-eqn-68"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,896)"><g id="mjx-eqn-69"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1505)"><g id="mjx-eqn-70"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-3266)"><g id="mjx-eqn-71"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-5511)"><g id="mjx-eqn-72"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-100">
\begin{align}
\nabla_WE=&\frac{\partial{E}}{\partial{W}}\\
=&\frac{\partial{E}}{\partial{\mathrm{net}_t}}\frac{\partial{\mathrm{net}_t}}{\partial{W}}\\
=&\nabla_{Wt}E+\delta_{t-1}^T\frac{\partial{\mathrm{net}_{t-1}}}{\partial{W}}\\
=&\nabla_{Wt}E+\nabla_{Wt-1}E+\delta_{t-2}^T\frac{\partial{\mathrm{net}_{t-2}}}{\partial{W}}\\
=&\nabla_{Wt}E+\nabla_{Wt-1}E+...+\nabla_{W1}E\\
=&\sum_{k=1}^t\nabla_{Wk}E
\end{align}
</script><p></p><p data-anchor-id="yk0g">这样，我们就证明了：最终的梯度<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-101-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 2439.401460074095 888.6635947032757" style="width: 5.685ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="1178" y="-213"></use><use href="#MJMATHI-45" x="1674" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-101">\nabla_WE</script>是各个时刻的梯度之和。</p><p data-anchor-id="qj15"><code>----------数学公式超高能预警解除----------</code></p><p data-anchor-id="yhhh">同权重矩阵W类似，我们可以得到权重矩阵U的计算方法。</p><div class="md-section-divider"></div><p data-anchor-id="dezv"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-102-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 46.331ex; height: 14.395ex; vertical-align: -6.653ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3375.246415789708 19956.94738405088 6250.492831579416" style="width: 46.331ex; height: 14.516ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-55"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-74" x="966" y="-213"></use></g><use href="#MJMATHI-45" x="1668" y="0"></use><use href="#MJMAIN-3D" x="2710" y="0"></use><g transform="translate(3766,0)"><g transform="translate(0,3356)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-4456.605784411571) scale(1,4.486908431859985)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-5568"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,2556)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-435"></use></g></g><g transform="translate(35,1051)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-435"></use></g></g><use href="#MJMAIN-2E" x="858" y="-149"></use><use href="#MJMAIN-2E" x="858" y="-1349"></use><g transform="translate(0,-2549)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-435"></use></g></g></g><g transform="translate(2981,0)"><g transform="translate(35,2556)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-435"></use></g></g><g transform="translate(35,1051)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-435"></use></g></g><g transform="translate(0,-2549)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-435"></use></g></g></g><g transform="translate(5976,0)"><g transform="translate(0,2556)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1051)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-2549)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(8145,0)"><g transform="translate(35,2556)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="809" y="-212"></use></g></g><g transform="translate(35,1051)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="628" y="-435"></use><g transform="translate(898,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="809" y="-212"></use></g></g><g transform="translate(0,-2549)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="641" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="628" y="-212"></use><g transform="translate(969,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="809" y="499"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="809" y="-212"></use></g></g></g></g><g transform="translate(11409,3356)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-4456.605784411571) scale(1,4.486908431859985)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-5568"></use></g></g><use href="#MJMAIN-28" x="17844" y="0"></use><g transform="translate(18233,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-38" x="19066" y="0"></use><use href="#MJMAIN-29" x="19567" y="0"></use></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-102">
\nabla_{U_t}E=\begin{bmatrix}
\delta_1^tx_1^t & \delta_1^tx_2^t & ... &  \delta_1^tx_m^t\\
\delta_2^tx_1^t & \delta_2^tx_2^t & ... &  \delta_2^tx_m^t\\
.\\.\\
\delta_n^tx_1^t & \delta_n^tx_2^t & ... &  \delta_n^tx_m^t\\
\end{bmatrix}\qquad(式8)
</script><p></p><p data-anchor-id="cc4o"><strong>式8</strong>是误差函数在t时刻对权重矩阵U的梯度。和权重矩阵W一样，最终的梯度也是各个时刻的梯度之和：</p><div class="md-section-divider"></div><p data-anchor-id="3t0e"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-103-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1713.5560211719578 7610.6948398425575 2930.547594965616" style="width: 17.661ex; height: 6.774ex; vertical-align: -2.903ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-2207"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-55" x="1178" y="-213"></use><use href="#MJMATHI-45" x="1476" y="0"></use><use href="#MJMAIN-3D" x="2518" y="0"></use><g transform="translate(3574,0)"><use href="#MJSZ2-2211"></use><g transform="translate(147,-1090)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-69"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-3D" x="345" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1124" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="840" y="1627"></use></g><g transform="translate(5185,0)"><use href="#MJMAIN-2207"></use><g transform="translate(833,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-55"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="966" y="-213"></use></g></g><use href="#MJMATHI-45" x="6846" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-103">
\nabla_UE=\sum_{i=1}^t\nabla_{U_i}E
</script><p></p><p data-anchor-id="023a">具体的证明这里就不再赘述了，感兴趣的读者可以练习推导一下。</p><div class="md-section-divider"></div><h3 data-anchor-id="h8ti" id="rnn的梯度爆炸和消失问题">RNN的梯度爆炸和消失问题</h3><p data-anchor-id="z8vf">不幸的是，实践中前面介绍的几种RNNs并不能很好的处理较长的序列。一个主要的原因是，RNN在训练中很容易发生<strong>梯度爆炸</strong>和<strong>梯度消失</strong>，这导致训练时梯度不能在较长序列中一直传递下去，从而使RNN无法捕捉到长距离的影响。</p><p data-anchor-id="m290">为什么RNN会产生梯度爆炸和消失问题呢？我们接下来将详细分析一下原因。我们根据<strong>式3</strong>可得：</p><div class="md-section-divider"></div><p data-anchor-id="4cbn"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-104-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -4111.8154854017 44270.673387096766 7723.6309708034" style="width: 102.823ex; height: 17.903ex; vertical-align: -8.468ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(14351,0)"><g transform="translate(-15,0)"><g transform="translate(1001,2369)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-463"></use><use href="#MJMAIN-3D" x="1329" y="0"></use></g><g transform="translate(0,-871)"><use href="#MJMAIN-2225"></use><g transform="translate(500,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-463"></use></g><use href="#MJMAIN-2225" x="1552" y="0"></use><use href="#MJAMS-2A7D" x="2330" y="0"></use></g><use href="#MJAMS-2A7D" x="2330" y="-3293"></use></g><g transform="translate(3372,0)"><g transform="translate(0,2369)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use><g transform="translate(1218,0)"><use href="#MJSZ2-220F"></use><g transform="translate(57,-1110)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-69"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-3D" x="345" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1124" y="0"></use></g><g transform="translate(59,1151)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMATHI-57" x="2663" y="0"></use><use href="#MJMATHI-64" x="3712" y="0"></use><use href="#MJMATHI-69" x="4235" y="0"></use><use href="#MJMATHI-61" x="4581" y="0"></use><use href="#MJMATHI-67" x="5110" y="0"></use><use href="#MJMAIN-5B" x="5591" y="0"></use><g transform="translate(5869,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="6732" y="0"></use><g transform="translate(7122,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="1966" y="-213"></use></g><use href="#MJMAIN-29" x="8857" y="0"></use><use href="#MJMAIN-5D" x="9246" y="0"></use></g><g transform="translate(0,-871)"><use href="#MJMAIN-2225"></use><g transform="translate(500,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g><use href="#MJMAIN-2225" x="1552" y="0"></use><g transform="translate(2219,0)"><use href="#MJSZ2-220F"></use><g transform="translate(57,-1110)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-69"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-3D" x="345" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1124" y="0"></use></g><g transform="translate(59,1151)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1140" y="0"></use></g></g><use href="#MJMAIN-2225" x="3664" y="0"></use><use href="#MJMATHI-57" x="4165" y="0"></use><use href="#MJMAIN-2225" x="5213" y="0"></use><use href="#MJMAIN-2225" x="5714" y="0"></use><use href="#MJMATHI-64" x="6214" y="0"></use><use href="#MJMATHI-69" x="6738" y="0"></use><use href="#MJMATHI-61" x="7083" y="0"></use><use href="#MJMATHI-67" x="7613" y="0"></use><use href="#MJMAIN-5B" x="8093" y="0"></use><g transform="translate(8372,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="9235" y="0"></use><g transform="translate(9624,0)"><use href="#MJMAIN-6E"></use><use href="#MJMAIN-65" x="556" y="0"></use><use href="#MJMAIN-74" x="1001" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="1966" y="-213"></use></g><use href="#MJMAIN-29" x="11359" y="0"></use><use href="#MJMAIN-5D" x="11749" y="0"></use><use href="#MJMAIN-2225" x="12027" y="0"></use></g><g transform="translate(0,-3293)"><use href="#MJMAIN-2225"></use><g transform="translate(500,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="641" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="628" y="-395"></use></g><use href="#MJMAIN-2225" x="1552" y="0"></use><use href="#MJMAIN-28" x="2052" y="0"></use><g transform="translate(2442,0)"><use href="#MJMATHI-3B2"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-57" x="801" y="-213"></use></g><g transform="translate(3850,0)"><use href="#MJMATHI-3B2"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-66" x="801" y="-219"></use></g><g transform="translate(4905,0)"><use href="#MJMAIN-29"></use><g transform="translate(389,412)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="361" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1140" y="0"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,2369)"><g id="mjx-eqn-73"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-871)"><g id="mjx-eqn-74"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-3293)"><g id="mjx-eqn-75"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-104">
\begin{align}
\delta_k^T=&\delta_t^T\prod_{i=k}^{t-1}Wdiag[f'(\mathrm{net}_{i})]\\
\|\delta_k^T\|\leqslant&\|\delta_t^T\|\prod_{i=k}^{t-1}\|W\|\|diag[f'(\mathrm{net}_{i})]\|\\
\leqslant&\|\delta_t^T\|(\beta_W\beta_f)^{t-k}
\end{align}
</script><p></p><p data-anchor-id="u32y">上式的<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-105-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -725.2000693679926 573.5 939.4001387359851" style="width: 1.331ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B2"></use></g></svg></span><script type="math/tex" id="MathJax-Element-105">\beta</script>定义为矩阵的模的上界。因为上式是一个指数函数，如果t-k很大的话（也就是向前看很远的时候），会导致对应的<strong>误差项</strong>的值增长或缩小的非常快，这样就会导致相应的<strong>梯度爆炸</strong>和<strong>梯度消失</strong>问题（取决于<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-106-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -725.2000693679926 573.5 939.4001387359851" style="width: 1.331ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B2"></use></g></svg></span><script type="math/tex" id="MathJax-Element-106">\beta</script>大于1还是小于1）。</p><p data-anchor-id="ej34">通常来说，<strong>梯度爆炸</strong>更容易处理一些。因为梯度爆炸的时候，我们的程序会收到NaN错误。我们也可以设置一个梯度阈值，当梯度超过这个阈值的时候可以直接截取。</p><p data-anchor-id="22xz"><strong>梯度消失</strong>更难检测，而且也更难处理一些。总的来说，我们有三种方法应对梯度消失问题：</p><ol data-anchor-id="g5nt">
<li>合理的初始化权重值。初始化权重，使每个神经元尽可能不要取极大或极小值，以躲开梯度消失的区域。</li>
<li>使用relu代替sigmoid和tanh作为激活函数。原理请参考上一篇文章<a href="https://www.zybuluo.com/hanbingtao/note/485480" target="_blank">零基础入门深度学习(4) - 卷积神经网络</a>的<strong>激活函数</strong>一节。</li>
<li>使用其他结构的RNNs，比如长短时记忆网络（LTSM）和Gated Recurrent Unit（GRU），这是最流行的做法。我们将在以后的文章中介绍这两种网络。</li>
</ol><div class="md-section-divider"></div><h2 data-anchor-id="cpiz" id="rnn的应用举例基于rnn的语言模型">RNN的应用举例——基于RNN的语言模型</h2><p data-anchor-id="0wa8">现在，我们介绍一下基于RNN语言模型。我们首先把词依次输入到循环神经网络中，每输入一个词，循环神经网络就输出截止到目前为止，下一个最可能的词。例如，当我们依次输入：</p><blockquote data-anchor-id="5q3y" class="white-blockquote">
  <p>我 昨天 上学 迟到 了</p>
</blockquote><p data-anchor-id="ay04">神经网络的输出如下图所示：</p><p data-anchor-id="e0l9"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-a69765380a75f860.png" alt="" title=""></p><p data-anchor-id="iqff">其中，s和e是两个特殊的词，分别表示一个序列的开始和结束。</p><div class="md-section-divider"></div><h3 data-anchor-id="cnga" id="向量化">向量化</h3><p data-anchor-id="lz0c">我们知道，神经网络的输入和输出都是<strong>向量</strong>，为了让语言模型能够被神经网络处理，我们必须把词表达为向量的形式，这样神经网络才能处理它。</p><p data-anchor-id="jrke">神经网络的输入是<strong>词</strong>，我们可以用下面的步骤对输入进行<strong>向量化</strong>：</p><ol data-anchor-id="qd8k">
<li>建立一个包含所有词的词典，每个词在词典里面有一个唯一的编号。</li>
<li>任意一个词都可以用一个N维的one-hot向量来表示。其中，N是词典中包含的词的个数。假设一个词在词典中的编号是i，v是表示这个词的向量，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-107-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 877.1815472394509 777.3570288792274" style="width: 2.056ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-76"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="686" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-107">v_j</script>是向量的第j个元素，则：</li>
</ol><div class="md-section-divider"></div><p data-anchor-id="y2ry"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-108-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1469.7000693679925 44270.673387096766 2439.400138735985" style="width: 102.823ex; height: 5.685ex; vertical-align: -2.298ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(18372,0)"><g transform="translate(-15,0)"><use href="#MJMATHI-76"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="686" y="-213"></use><use href="#MJMAIN-3D" x="1154" y="0"></use><g transform="translate(2211,0)"><use href="#MJSZ3-7B"></use><g transform="translate(917,0)"><g transform="translate(-15,0)"><g transform="translate(0,558)"><use href="#MJMAIN-31"></use><use href="#MJMATHI-6A" x="2500" y="0"></use><use href="#MJMAIN-3D" x="3190" y="0"></use><use href="#MJMATHI-69" x="4247" y="0"></use></g><g transform="translate(0,-642)"><use href="#MJMAIN-30"></use><use href="#MJMATHI-6A" x="2500" y="0"></use><use href="#MJMAIN-2260" x="3190" y="0"></use><use href="#MJMATHI-69" x="4247" y="0"></use></g></g></g></g></g></g><g transform="translate(41690,0)"><g id="mjx-eqn-76"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-108">
v_j=\begin{equation}\begin{cases}1\qquad j=i\\0\qquad j\ne i\end{cases}\end{equation}
</script><p></p><p data-anchor-id="aly3">上面这个公式的含义，可以用下面的图来直观的表示：</p><p data-anchor-id="jfi3"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-14ae8b4f92e90c5c.png" alt="" title=""></p><p data-anchor-id="6zhn">使用这种向量化方法，我们就得到了一个高维、<strong>稀疏</strong>的向量（稀疏是指绝大部分元素的值都是0）。处理这样的向量会导致我们的神经网络有很多的参数，带来庞大的计算量。因此，往往会需要使用一些降维方法，将高维的稀疏向量转变为低维的稠密向量。不过这个话题我们就不再这篇文章中讨论了。</p><p data-anchor-id="ti80">语言模型要求的输出是下一个最可能的词，我们可以让循环神经网络计算计算词典中每个词是下一个词的概率，这样，概率最大的词就是下一个最可能的词。因此，神经网络的输出向量也是一个N维向量，向量中的每个元素对应着词典中相应的词是下一个词的概率。如下图所示：</p><p data-anchor-id="7n66"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-3e1562c7031309f1.png" alt="" title=""></p><div class="md-section-divider"></div><h3 data-anchor-id="okcw" id="softmax层">Softmax层</h3><p data-anchor-id="ysrn">前面提到，<strong>语言模型</strong>是对下一个词出现的<strong>概率</strong>进行建模。那么，怎样让神经网络输出概率呢？方法就是用softmax层作为神经网络的输出层。</p><p data-anchor-id="3ljy">我们先来看一下softmax函数的定义：</p><div class="md-section-divider"></div><p data-anchor-id="cq12"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-109-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1371.848373433633 6682.402686271951 2407.096926138417" style="width: 15.484ex; height: 5.565ex; vertical-align: -2.54ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-67"></use><use href="#MJMAIN-28" x="480" y="0"></use><g transform="translate(870,0)"><use href="#MJMATHI-7A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="658" y="-213"></use></g><use href="#MJMAIN-29" x="1679" y="0"></use><use href="#MJMAIN-3D" x="2347" y="0"></use><g transform="translate(3523,0)"><rect stroke="none" width="3039" height="60" x="0" y="220"></rect><g transform="translate(949,676)"><use href="#MJMATHI-65"></use><g transform="translate(466,362)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-7A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="658" y="-213"></use></g></g><g transform="translate(60,-722)"><use href="#MJSZ1-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1494" y="-405"></use><g transform="translate(1691,0)"><use href="#MJMATHI-65"></use><g transform="translate(466,288)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-7A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6B" x="658" y="-213"></use></g></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-109">
g(z_i)=\frac{e^{z_i}}{\sum_{k}e^{z_k}}
</script><p></p><p data-anchor-id="w9vm">这个公式看起来可能很晕，我们举一个例子。Softmax层如下图所示：</p><p data-anchor-id="5nqq"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-5a3219fab80ab45f.png" alt="" title=""></p><p data-anchor-id="c3sb">从上图我们可以看到，softmax layer的输入是一个向量，输出也是一个向量，两个向量的维度是一样的（在这个例子里面是4）。输入向量x=[1 2 3 4]经过softmax层之后，经过上面的softmax函数计算，转变为输出向量y=[0.03 0.09 0.24 0.64]。计算过程为：</p><div class="md-section-divider"></div><p data-anchor-id="anca"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-110-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -9326.192964173408 44270.673387096766 18152.385928346815" style="width: 102.823ex; height: 42.218ex; vertical-align: -20.565ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17314,0)"><g transform="translate(-15,0)"><g transform="translate(0,7954)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g><g transform="translate(0,1192)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="693" y="-213"></use></g><g transform="translate(0,-2742)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="693" y="-213"></use></g><g transform="translate(0,-6684)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="693" y="-213"></use></g></g><g transform="translate(1208,0)"><g transform="translate(0,7954)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="3114" height="60" x="0" y="220"></rect><g transform="translate(911,676)"><use href="#MJMATHI-65"></use><g transform="translate(466,362)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-78"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="809" y="-213"></use></g></g><g transform="translate(60,-722)"><use href="#MJSZ1-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="1494" y="-405"></use><g transform="translate(1691,0)"><use href="#MJMATHI-65"></use><g transform="translate(466,288)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-78"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6B" x="809" y="-213"></use></g></g></g></g></g><g transform="translate(0,5127)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="7470" height="60" x="0" y="220"></rect><g transform="translate(3275,676)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="659" y="513"></use></g><g transform="translate(60,-739)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="659" y="408"></use><use href="#MJMAIN-2B" x="1142" y="0"></use><g transform="translate(2143,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="659" y="408"></use></g><use href="#MJMAIN-2B" x="3285" y="0"></use><g transform="translate(4286,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="659" y="408"></use></g><use href="#MJMAIN-2B" x="5429" y="0"></use><g transform="translate(6430,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="659" y="408"></use></g></g></g></g><g transform="translate(0,3254)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-30" x="779" y="0"></use><use href="#MJMAIN-33" x="1279" y="0"></use></g></g><g transform="translate(0,1192)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="7470" height="60" x="0" y="220"></rect><g transform="translate(3275,676)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="659" y="513"></use></g><g transform="translate(60,-739)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="659" y="408"></use><use href="#MJMAIN-2B" x="1142" y="0"></use><g transform="translate(2143,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="659" y="408"></use></g><use href="#MJMAIN-2B" x="3285" y="0"></use><g transform="translate(4286,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="659" y="408"></use></g><use href="#MJMAIN-2B" x="5429" y="0"></use><g transform="translate(6430,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="659" y="408"></use></g></g></g></g><g transform="translate(0,-680)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-30" x="779" y="0"></use><use href="#MJMAIN-39" x="1279" y="0"></use></g></g><g transform="translate(0,-2742)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="7470" height="60" x="0" y="220"></rect><g transform="translate(3275,676)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="659" y="513"></use></g><g transform="translate(60,-739)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="659" y="408"></use><use href="#MJMAIN-2B" x="1142" y="0"></use><g transform="translate(2143,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="659" y="408"></use></g><use href="#MJMAIN-2B" x="3285" y="0"></use><g transform="translate(4286,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="659" y="408"></use></g><use href="#MJMAIN-2B" x="5429" y="0"></use><g transform="translate(6430,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="659" y="408"></use></g></g></g></g><g transform="translate(0,-4614)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-32" x="779" y="0"></use><use href="#MJMAIN-34" x="1279" y="0"></use></g></g><g transform="translate(0,-6684)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="7470" height="60" x="0" y="220"></rect><g transform="translate(3275,676)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="659" y="513"></use></g><g transform="translate(60,-739)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="659" y="408"></use><use href="#MJMAIN-2B" x="1142" y="0"></use><g transform="translate(2143,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="659" y="408"></use></g><use href="#MJMAIN-2B" x="3285" y="0"></use><g transform="translate(4286,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="659" y="408"></use></g><use href="#MJMAIN-2B" x="5429" y="0"></use><g transform="translate(6430,0)"><use href="#MJMATHI-65"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="659" y="408"></use></g></g></g></g><g transform="translate(0,-8556)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-36" x="779" y="0"></use><use href="#MJMAIN-34" x="1279" y="0"></use></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,7954)"><g id="mjx-eqn-77"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,5127)"><g id="mjx-eqn-78"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,3254)"><g id="mjx-eqn-79"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,1192)"><g id="mjx-eqn-80"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-680)"><g id="mjx-eqn-81"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2742)"><g id="mjx-eqn-82"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-4614)"><g id="mjx-eqn-83"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-6684)"><g id="mjx-eqn-84"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-8556)"><g id="mjx-eqn-85"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-110">
\begin{align}
y_1&=\frac{e^{x_1}}{\sum_{k}e^{x_k}}\\
&=\frac{e^1}{e^1+e^2+e^3+e^4}\\
&=0.03\\
y_2&=\frac{e^2}{e^1+e^2+e^3+e^4}\\
&=0.09\\
y_3&=\frac{e^3}{e^1+e^2+e^3+e^4}\\
&=0.24\\
y_4&=\frac{e^4}{e^1+e^2+e^3+e^4}\\
&=0.64\\
\end{align}
</script><p></p><p data-anchor-id="isy4">我们来看看输出向量y的特征：</p><ol data-anchor-id="erny">
<li>每一项为取值为0-1之间的正数；</li>
<li>所有项的总和是1。</li>
</ol><p data-anchor-id="p49n">我们不难发现，这些特征和<strong>概率</strong>的特征是一样的，因此我们可以把它们看做是概率。对于<strong>语言模型</strong>来说，我们可以认为模型预测下一个词是词典中第一个词的概率是0.03，是词典中第二个词的概率是0.09，以此类推。</p><div class="md-section-divider"></div><h3 data-anchor-id="f13j" id="语言模型的训练">语言模型的训练</h3><p data-anchor-id="ejtc">可以使用<strong>监督学习</strong>的方法对语言模型进行训练，首先，需要准备训练数据集。接下来，我们介绍怎样把语料</p><blockquote data-anchor-id="oblq" class="white-blockquote">
  <p>我 昨天 上学 迟到 了</p>
</blockquote><p data-anchor-id="ppiv">转换成语言模型的训练数据集。</p><p data-anchor-id="9qeu">首先，我们获取<strong>输入-标签</strong>对：</p><table data-anchor-id="axjw" class="table table-striped-white table-bordered">
<thead>
<tr>
 <th style="text-align:center;">输入</th>
 <th style="text-align:center;">标签</th>
</tr>
</thead>
<tbody><tr>
 <td style="text-align:center;">s</td>
 <td style="text-align:center;">我</td>
</tr>
<tr>
 <td style="text-align:center;">我</td>
 <td style="text-align:center;">昨天</td>
</tr>
<tr>
 <td style="text-align:center;">昨天</td>
 <td style="text-align:center;">上学</td>
</tr>
<tr>
 <td style="text-align:center;">上学</td>
 <td style="text-align:center;">迟到</td>
</tr>
<tr>
 <td style="text-align:center;">迟到</td>
 <td style="text-align:center;">了</td>
</tr>
<tr>
 <td style="text-align:center;">了</td>
 <td style="text-align:center;">e</td>
</tr>
</tbody></table><p data-anchor-id="j4lz">然后，使用前面介绍过的<strong>向量化</strong>方法，对输入x和标签y进行<strong>向量化</strong>。这里面有意思的是，对标签y进行向量化，其结果也是一个one-hot向量。例如，我们对标签『我』进行向量化，得到的向量中，只有第2019个元素的值是1，其他位置的元素的值都是0。它的含义就是下一个词是『我』的概率是1，是其它词的概率都是0。</p><p data-anchor-id="qf8o">最后，我们使用<strong>交叉熵误差函数</strong>作为优化目标，对模型进行优化。</p><p data-anchor-id="golp">在实际工程中，我们可以使用大量的语料来对模型进行训练，获取训练数据和训练的方法都是相同的。</p><div class="md-section-divider"></div><h3 data-anchor-id="x4o8" id="交叉熵误差">交叉熵误差</h3><p data-anchor-id="no1f">一般来说，当神经网络的输出层是softmax层时，对应的误差函数E通常选择交叉熵误差函数，其定义如下：</p><div class="md-section-divider"></div><p data-anchor-id="2rn4"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-111-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1362.7080693679925 11564.333240056056 2612.9336618774187" style="width: 26.855ex; height: 6.048ex; vertical-align: -3.024ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-4C"></use><use href="#MJMAIN-28" x="681" y="0"></use><use href="#MJMATHI-79" x="1071" y="0"></use><use href="#MJMAIN-2C" x="1568" y="0"></use><use href="#MJMATHI-6F" x="2013" y="0"></use><use href="#MJMAIN-29" x="2499" y="0"></use><use href="#MJMAIN-3D" x="3166" y="0"></use><use href="#MJMAIN-2212" x="4222" y="0"></use><g transform="translate(5287,0)"><rect stroke="none" width="1008" height="60" x="0" y="220"></rect><use href="#MJMAIN-31" x="254" y="676"></use><use href="#MJMATHI-4E" x="60" y="-686"></use></g><g transform="translate(6583,0)"><use href="#MJSZ2-2211" x="40" y="0"></use><g transform="translate(0,-1102)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="600" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-4E" x="1268" y="0"></use></g></g><g transform="translate(8274,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="693" y="-213"></use><use href="#MJMATHI-6C" x="1015" y="0"></use><use href="#MJMATHI-6F" x="1313" y="0"></use><use href="#MJMATHI-67" x="1799" y="0"></use><g transform="translate(2279,0)"><use href="#MJMATHI-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="686" y="-213"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-111">
L(y,o)=-\frac{1}{N}\sum_{n\in{N}}{y_nlogo_n}
</script><p></p><p data-anchor-id="6ga3">在上式中，N是训练样本的个数，向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-112-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1015.1176221025219 688.4001387359851" style="width: 2.298ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-112">y_n</script>是样本的标记，向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-113-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 1010.1176221025219 638.8854201102238" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="686" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-113">o_n</script>是网络的输出。标记<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-114-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1015.1176221025219 688.4001387359851" style="width: 2.298ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-114">y_n</script>是一个one-hot向量，例如<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-115-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 6172.962499539422 1040.4001387359851" style="width: 14.395ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use><use href="#MJMAIN-3D" x="1222" y="0"></use><use href="#MJMAIN-5B" x="2278" y="0"></use><use href="#MJMAIN-31" x="2556" y="0"></use><use href="#MJMAIN-2C" x="3057" y="0"></use><use href="#MJMAIN-30" x="3502" y="0"></use><use href="#MJMAIN-2C" x="4003" y="0"></use><use href="#MJMAIN-30" x="4448" y="0"></use><use href="#MJMAIN-2C" x="4948" y="0"></use><use href="#MJMAIN-30" x="5393" y="0"></use><use href="#MJMAIN-5D" x="5894" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-115">y_1=[1,0,0,0]</script>，如果网络的输出<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-116-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 10832.055555555557 1040.4001387359851" style="width: 25.161ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6F"></use><use href="#MJMAIN-3D" x="763" y="0"></use><use href="#MJMAIN-5B" x="1819" y="0"></use><g transform="translate(2098,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-30" x="779" y="0"></use><use href="#MJMAIN-33" x="1279" y="0"></use></g><use href="#MJMAIN-2C" x="3878" y="0"></use><g transform="translate(4323,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-30" x="779" y="0"></use><use href="#MJMAIN-39" x="1279" y="0"></use></g><use href="#MJMAIN-2C" x="6103" y="0"></use><g transform="translate(6548,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-32" x="779" y="0"></use><use href="#MJMAIN-34" x="1279" y="0"></use></g><use href="#MJMAIN-2C" x="8328" y="0"></use><g transform="translate(8773,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-36" x="779" y="0"></use><use href="#MJMAIN-34" x="1279" y="0"></use></g><use href="#MJMAIN-5D" x="10553" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-116">o=[0.03,0.09,0.24,0.64]</script>，那么，交叉熵误差是（假设只有一个训练样本，即N=1）：</p><div class="md-section-divider"></div><p data-anchor-id="4gw2"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-117-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3509.4668309387093 44270.673387096766 6518.933661877419" style="width: 102.823ex; height: 15.121ex; vertical-align: -7.137ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(9708,0)"><g transform="translate(-15,0)"><use href="#MJMATHI-4C" x="0" y="2146"></use></g><g transform="translate(945,0)"><g transform="translate(0,2146)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-2212" x="1056" y="0"></use><g transform="translate(2121,0)"><rect stroke="none" width="1008" height="60" x="0" y="220"></rect><use href="#MJMAIN-31" x="254" y="676"></use><use href="#MJMATHI-4E" x="60" y="-686"></use></g><g transform="translate(3416,0)"><use href="#MJSZ2-2211" x="40" y="0"></use><g transform="translate(0,-1102)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="600" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-4E" x="1268" y="0"></use></g></g><g transform="translate(5108,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="693" y="-213"></use><use href="#MJMATHI-6C" x="1015" y="0"></use><use href="#MJMATHI-6F" x="1313" y="0"></use><use href="#MJMATHI-67" x="1799" y="0"></use><g transform="translate(2279,0)"><use href="#MJMATHI-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="686" y="-213"></use></g></g></g><g transform="translate(0,-136)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-2212" x="1056" y="0"></use><g transform="translate(1834,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g><use href="#MJMATHI-6C" x="2779" y="0"></use><use href="#MJMATHI-6F" x="3077" y="0"></use><use href="#MJMATHI-67" x="3563" y="0"></use><g transform="translate(4043,0)"><use href="#MJMATHI-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="686" y="-213"></use></g></g><g transform="translate(0,-1438)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-2212" x="1056" y="0"></use><use href="#MJMAIN-28" x="1834" y="0"></use><use href="#MJMAIN-31" x="2224" y="0"></use><use href="#MJMAIN-2217" x="2947" y="0"></use><use href="#MJMATHI-6C" x="3669" y="0"></use><use href="#MJMATHI-6F" x="3968" y="0"></use><use href="#MJMATHI-67" x="4453" y="0"></use><g transform="translate(4934,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-30" x="779" y="0"></use><use href="#MJMAIN-33" x="1279" y="0"></use></g><use href="#MJMAIN-2B" x="6936" y="0"></use><use href="#MJMAIN-30" x="7937" y="0"></use><use href="#MJMAIN-2217" x="8659" y="0"></use><use href="#MJMATHI-6C" x="9382" y="0"></use><use href="#MJMATHI-6F" x="9681" y="0"></use><use href="#MJMATHI-67" x="10166" y="0"></use><g transform="translate(10647,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-30" x="779" y="0"></use><use href="#MJMAIN-39" x="1279" y="0"></use></g><use href="#MJMAIN-2B" x="12649" y="0"></use><use href="#MJMAIN-30" x="13650" y="0"></use><use href="#MJMAIN-2217" x="14372" y="0"></use><use href="#MJMATHI-6C" x="15095" y="0"></use><use href="#MJMATHI-6F" x="15394" y="0"></use><use href="#MJMATHI-67" x="15879" y="0"></use><g transform="translate(16360,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-32" x="779" y="0"></use><use href="#MJMAIN-34" x="1279" y="0"></use></g><use href="#MJMAIN-2B" x="18362" y="0"></use><use href="#MJMAIN-30" x="19362" y="0"></use><use href="#MJMAIN-2217" x="20085" y="0"></use><use href="#MJMATHI-6C" x="20808" y="0"></use><use href="#MJMATHI-6F" x="21106" y="0"></use><use href="#MJMATHI-67" x="21592" y="0"></use><g transform="translate(22072,0)"><use href="#MJMAIN-30"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-36" x="779" y="0"></use><use href="#MJMAIN-34" x="1279" y="0"></use></g><use href="#MJMAIN-29" x="23852" y="0"></use></g><g transform="translate(0,-2740)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMAIN-33"></use><use href="#MJMAIN-2E" x="500" y="0"></use><use href="#MJMAIN-35" x="779" y="0"></use><use href="#MJMAIN-31" x="1279" y="0"></use></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,2146)"><g id="mjx-eqn-86"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-136)"><g id="mjx-eqn-87"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1438)"><g id="mjx-eqn-88"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2740)"><g id="mjx-eqn-89"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-117">
\begin{align}
L&=-\frac{1}{N}\sum_{n\in{N}}{y_nlogo_n}\\
&=-y_1logo_1\\
&=-(1*log0.03+0*log0.09+0*log0.24+0*log0.64)\\
&=3.51
\end{align}
</script><p></p><p data-anchor-id="g7eu">我们当然可以选择其他函数作为我们的误差函数，比如最小平方误差函数(MSE)。不过对概率进行建模时，选择交叉熵误差函数更make sense。具体原因，感兴趣的读者请阅读<a href="https://jamesmccaffrey.wordpress.com/2011/12/17/neural-network-classification-categorical-data-softmax-activation-and-cross-entropy-error/" target="_blank">参考文献7</a>。</p><div class="md-section-divider"></div><h2 data-anchor-id="qprl" id="rnn的实现">RNN的实现</h2><blockquote data-anchor-id="bk5i" class="white-blockquote">
  <p>完整代码请参考GitHub: <a href="https://github.com/hanbt/learn_dl/blob/master/rnn.py" target="_blank">https://github.com/hanbt/learn_dl/blob/master/rnn.py</a> (python2.7)</p>
</blockquote><p data-anchor-id="rrwv">为了加深我们对前面介绍的知识的理解，我们来动手实现一个RNN层。我们复用了上一篇文章<a href="https://www.zybuluo.com/hanbingtao/note/485480" target="_blank">零基础入门深度学习(4) - 卷积神经网络</a>中的一些代码，所以先把它们导入进来。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="v225" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">import</span><span class="pln"> numpy </span><span class="kwd">as</span><span class="pln"> np</span></code></li><li class="L1"><code class="language-python"><span class="kwd">from</span><span class="pln"> cnn </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">ReluActivator</span><span class="pun">,</span><span class="pln"> </span><span class="typ">IdentityActivator</span><span class="pun">,</span><span class="pln"> element_wise_op</span></code></li></ol></pre><p data-anchor-id="f8i8">我们用RecurrentLayer类来实现一个<strong>循环层</strong>。下面的代码是初始化一个循环层，可以在构造函数中设置卷积层的超参数。我们注意到，循环层有两个权重数组，U和W。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="uawb" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">RecurrentLayer</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> input_width</span><span class="pun">,</span><span class="pln"> state_width</span><span class="pun">,</span></code></li><li class="L2"><code class="language-python"><span class="pln">                 activator</span><span class="pun">,</span><span class="pln"> learning_rate</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">input_width </span><span class="pun">=</span><span class="pln"> input_width</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">state_width </span><span class="pun">=</span><span class="pln"> state_width</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">activator </span><span class="pun">=</span><span class="pln"> activator</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">learning_rate </span><span class="pun">=</span><span class="pln"> learning_rate</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">times </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pln">       </span><span class="com"># 当前时刻初始化为t0</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">state_list </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span><span class="pln"> </span><span class="com"># 保存各个时刻的state</span></code></li><li class="L0"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">(</span></code></li><li class="L1"><code class="language-python"><span class="pln">            </span><span class="pun">(</span><span class="pln">state_width</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)))</span><span class="pln">           </span><span class="com"># 初始化s0</span></code></li><li class="L2"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">U </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">random</span><span class="pun">.</span><span class="pln">uniform</span><span class="pun">(-</span><span class="lit">1e-4</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1e-4</span><span class="pun">,</span></code></li><li class="L3"><code class="language-python"><span class="pln">            </span><span class="pun">(</span><span class="pln">state_width</span><span class="pun">,</span><span class="pln"> input_width</span><span class="pun">))</span><span class="pln">  </span><span class="com"># 初始化U</span></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">W </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">random</span><span class="pun">.</span><span class="pln">uniform</span><span class="pun">(-</span><span class="lit">1e-4</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1e-4</span><span class="pun">,</span></code></li><li class="L5"><code class="language-python"><span class="pln">            </span><span class="pun">(</span><span class="pln">state_width</span><span class="pun">,</span><span class="pln"> state_width</span><span class="pun">))</span><span class="pln">  </span><span class="com"># 初始化W</span></code></li></ol></pre><p data-anchor-id="bjw7">在forward方法中，实现循环层的前向计算，这部分比较简单。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="kelm" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> forward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> input_array</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        根据『式2』进行前向计算</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">times </span><span class="pun">+=</span><span class="pln"> </span><span class="lit">1</span></code></li><li class="L5"><code class="language-python"><span class="pln">        state </span><span class="pun">=</span><span class="pln"> </span><span class="pun">(</span><span class="pln">np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">U</span><span class="pun">,</span><span class="pln"> input_array</span><span class="pun">)</span><span class="pln"> </span><span class="pun">+</span></code></li><li class="L6"><code class="language-python"><span class="pln">                 np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">W</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">]))</span></code></li><li class="L7"><code class="language-python"><span class="pln">        element_wise_op</span><span class="pun">(</span><span class="pln">state</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">activator</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">)</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">state</span><span class="pun">)</span></code></li></ol></pre><p data-anchor-id="y3pl">在backword方法中，实现BPTT算法。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="kez1" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> backward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> sensitivity_array</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L1"><code class="language-python"><span class="pln">                 activator</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        实现BPTT算法</span></code></li><li class="L4"><code class="language-python"><span class="str">        '''</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">calc_delta</span><span class="pun">(</span><span class="pln">sensitivity_array</span><span class="pun">,</span><span class="pln"> activator</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">calc_gradient</span><span class="pun">()</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_delta</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> sensitivity_array</span><span class="pun">,</span><span class="pln"> activator</span><span class="pun">):</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">delta_list </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span><span class="pln">  </span><span class="com"># 用来保存各个时刻的误差项</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">times</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">delta_list</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">(</span></code></li><li class="L2"><code class="language-python"><span class="pln">                </span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">state_width</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)))</span></code></li><li class="L3"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">delta_list</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">sensitivity_array</span><span class="pun">)</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="com"># 迭代计算每个时刻的误差项</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> k </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">times </span><span class="pun">-</span><span class="pln"> </span><span class="lit">1</span><span class="pun">,</span><span class="pln"> </span><span class="lit">0</span><span class="pun">,</span><span class="pln"> </span><span class="pun">-</span><span class="lit">1</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">calc_delta_k</span><span class="pun">(</span><span class="pln">k</span><span class="pun">,</span><span class="pln"> activator</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_delta_k</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> k</span><span class="pun">,</span><span class="pln"> activator</span><span class="pun">):</span></code></li><li class="L9"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L0"><code class="language-python"><span class="str">        根据k+1时刻的delta计算k时刻的delta</span></code></li><li class="L1"><code class="language-python"><span class="str">        '''</span></code></li><li class="L2"><code class="language-python"><span class="pln">        state </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">[</span><span class="pln">k</span><span class="pun">+</span><span class="lit">1</span><span class="pun">].</span><span class="pln">copy</span><span class="pun">()</span></code></li><li class="L3"><code class="language-python"><span class="pln">        element_wise_op</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">[</span><span class="pln">k</span><span class="pun">+</span><span class="lit">1</span><span class="pun">],</span></code></li><li class="L4"><code class="language-python"><span class="pln">                    activator</span><span class="pun">.</span><span class="pln">backward</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">delta_list</span><span class="pun">[</span><span class="pln">k</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span></code></li><li class="L6"><code class="language-python"><span class="pln">            np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">delta_list</span><span class="pun">[</span><span class="pln">k</span><span class="pun">+</span><span class="lit">1</span><span class="pun">].</span><span class="pln">T</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">W</span><span class="pun">),</span></code></li><li class="L7"><code class="language-python"><span class="pln">            np</span><span class="pun">.</span><span class="pln">diag</span><span class="pun">(</span><span class="pln">state</span><span class="pun">[:,</span><span class="lit">0</span><span class="pun">])).</span><span class="pln">T</span></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_gradient</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L0"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">gradient_list </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span><span class="pln"> </span><span class="com"># 保存各个时刻的权重梯度</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> t </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">times </span><span class="pun">+</span><span class="pln"> </span><span class="lit">1</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">gradient_list</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">(</span></code></li><li class="L3"><code class="language-python"><span class="pln">                </span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">state_width</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">state_width</span><span class="pun">)))</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> t </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">times</span><span class="pun">,</span><span class="pln"> </span><span class="lit">0</span><span class="pun">,</span><span class="pln"> </span><span class="pun">-</span><span class="lit">1</span><span class="pun">):</span></code></li><li class="L5"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="typ">calc_gradient_t</span><span class="pun">(</span><span class="pln">t</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="com"># 实际的梯度是各个时刻梯度之和</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">gradient </span><span class="pun">=</span><span class="pln"> reduce</span><span class="pun">(</span></code></li><li class="L8"><code class="language-python"><span class="pln">            </span><span class="kwd">lambda</span><span class="pln"> a</span><span class="pun">,</span><span class="pln"> b</span><span class="pun">:</span><span class="pln"> a </span><span class="pun">+</span><span class="pln"> b</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">gradient_list</span><span class="pun">,</span></code></li><li class="L9"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">gradient_list</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="com"># [0]被初始化为0且没有被修改过</span></code></li><li class="L0"><code class="language-python"></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> </span><span class="typ">calc_gradient_t</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> t</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        计算每个时刻t权重的梯度</span></code></li><li class="L4"><code class="language-python"><span class="str">        '''</span></code></li><li class="L5"><code class="language-python"><span class="pln">        gradient </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">delta_list</span><span class="pun">[</span><span class="pln">t</span><span class="pun">],</span></code></li><li class="L6"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">[</span><span class="pln">t</span><span class="pun">-</span><span class="lit">1</span><span class="pun">].</span><span class="pln">T</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">gradient_list</span><span class="pun">[</span><span class="pln">t</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> gradient</span></code></li></ol></pre><p data-anchor-id="f5fe">有意思的是，BPTT算法虽然数学推导的过程很麻烦，但是写成代码却并不复杂。</p><p data-anchor-id="0rum">在update方法中，实现梯度下降算法。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="i7kc" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> update</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        按照梯度下降，更新权重</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">W </span><span class="pun">-=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">learning_rate </span><span class="pun">*</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">gradient</span></code></li></ol></pre><p data-anchor-id="d8d8">上面的代码不包含权重U的更新。这部分实际上和全连接神经网络是一样的，留给感兴趣的读者自己来完成吧。</p><p data-anchor-id="8hjo"><strong>循环层</strong>是一个<strong>带状态</strong>的层，每次forword都会改变循环层的内部状态，这给梯度检查带来了麻烦。因此，我们需要一个reset_state方法，来重置循环层的内部状态。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="yfb9" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> reset_state</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">times </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pln">       </span><span class="com"># 当前时刻初始化为t0</span></code></li><li class="L2"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">state_list </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span><span class="pln"> </span><span class="com"># 保存各个时刻的state</span></code></li><li class="L3"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">(</span></code></li><li class="L4"><code class="language-python"><span class="pln">            </span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">state_width</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)))</span><span class="pln">      </span><span class="com"># 初始化s0</span></code></li></ol></pre><p data-anchor-id="jvd1">最后，是梯度检查的代码。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="5jku" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">def</span><span class="pln"> gradient_check</span><span class="pun">():</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">    梯度检查</span></code></li><li class="L3"><code class="language-python"><span class="str">    '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="com"># 设计一个误差函数，取所有节点输出项之和</span></code></li><li class="L5"><code class="language-python"><span class="pln">    error_function </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">lambda</span><span class="pln"> o</span><span class="pun">:</span><span class="pln"> o</span><span class="pun">.</span><span class="pln">sum</span><span class="pun">()</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"><span class="pln">    rl </span><span class="pun">=</span><span class="pln"> </span><span class="typ">RecurrentLayer</span><span class="pun">(</span><span class="lit">3</span><span class="pun">,</span><span class="pln"> </span><span class="lit">2</span><span class="pun">,</span><span class="pln"> </span><span class="typ">IdentityActivator</span><span class="pun">(),</span><span class="pln"> </span><span class="lit">1e-3</span><span class="pun">)</span></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="com"># 计算forward值</span></code></li><li class="L0"><code class="language-python"><span class="pln">    x</span><span class="pun">,</span><span class="pln"> d </span><span class="pun">=</span><span class="pln"> data_set</span><span class="pun">()</span></code></li><li class="L1"><code class="language-python"><span class="pln">    rl</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span></code></li><li class="L2"><code class="language-python"><span class="pln">    rl</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="com"># 求取sensitivity map</span></code></li><li class="L5"><code class="language-python"><span class="pln">    sensitivity_array </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">ones</span><span class="pun">(</span><span class="pln">rl</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">].</span><span class="pln">shape</span><span class="pun">,</span></code></li><li class="L6"><code class="language-python"><span class="pln">                                dtype</span><span class="pun">=</span><span class="pln">np</span><span class="pun">.</span><span class="pln">float64</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="com"># 计算梯度</span></code></li><li class="L8"><code class="language-python"><span class="pln">    rl</span><span class="pun">.</span><span class="pln">backward</span><span class="pun">(</span><span class="pln">sensitivity_array</span><span class="pun">,</span><span class="pln"> </span><span class="typ">IdentityActivator</span><span class="pun">())</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="com"># 检查梯度</span></code></li><li class="L1"><code class="language-python"><span class="pln">    epsilon </span><span class="pun">=</span><span class="pln"> </span><span class="lit">10e-4</span></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">rl</span><span class="pun">.</span><span class="pln">W</span><span class="pun">.</span><span class="pln">shape</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> j </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">rl</span><span class="pun">.</span><span class="pln">W</span><span class="pun">.</span><span class="pln">shape</span><span class="pun">[</span><span class="lit">1</span><span class="pun">]):</span></code></li><li class="L4"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">W</span><span class="pun">[</span><span class="pln">i</span><span class="pun">,</span><span class="pln">j</span><span class="pun">]</span><span class="pln"> </span><span class="pun">+=</span><span class="pln"> epsilon</span></code></li><li class="L5"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">reset_state</span><span class="pun">()</span></code></li><li class="L6"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span></code></li><li class="L7"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L8"><code class="language-python"><span class="pln">            err1 </span><span class="pun">=</span><span class="pln"> error_function</span><span class="pun">(</span><span class="pln">rl</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L9"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">W</span><span class="pun">[</span><span class="pln">i</span><span class="pun">,</span><span class="pln">j</span><span class="pun">]</span><span class="pln"> </span><span class="pun">-=</span><span class="pln"> </span><span class="lit">2</span><span class="pun">*</span><span class="pln">epsilon</span></code></li><li class="L0"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">reset_state</span><span class="pun">()</span></code></li><li class="L1"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span></code></li><li class="L2"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L3"><code class="language-python"><span class="pln">            err2 </span><span class="pun">=</span><span class="pln"> error_function</span><span class="pun">(</span><span class="pln">rl</span><span class="pun">.</span><span class="pln">state_list</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L4"><code class="language-python"><span class="pln">            expect_grad </span><span class="pun">=</span><span class="pln"> </span><span class="pun">(</span><span class="pln">err1 </span><span class="pun">-</span><span class="pln"> err2</span><span class="pun">)</span><span class="pln"> </span><span class="pun">/</span><span class="pln"> </span><span class="pun">(</span><span class="lit">2</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> epsilon</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"><span class="pln">            rl</span><span class="pun">.</span><span class="pln">W</span><span class="pun">[</span><span class="pln">i</span><span class="pun">,</span><span class="pln">j</span><span class="pun">]</span><span class="pln"> </span><span class="pun">+=</span><span class="pln"> epsilon</span></code></li><li class="L6"><code class="language-python"><span class="pln">            </span><span class="kwd">print</span><span class="pln"> </span><span class="str">'weights(%d,%d): expected - actural %f - %f'</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span></code></li><li class="L7"><code class="language-python"><span class="pln">                i</span><span class="pun">,</span><span class="pln"> j</span><span class="pun">,</span><span class="pln"> expect_grad</span><span class="pun">,</span><span class="pln"> rl</span><span class="pun">.</span><span class="pln">gradient</span><span class="pun">[</span><span class="pln">i</span><span class="pun">,</span><span class="pln">j</span><span class="pun">])</span></code></li></ol></pre><p data-anchor-id="owl3">需要注意，每次计算error之前，都要调用reset_state方法重置循环层的内部状态。下面是梯度检查的结果，没问题！</p><p data-anchor-id="66ha"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-1bdfa618b5dbaabd.png" alt="" title=""></p><div class="md-section-divider"></div><h2 data-anchor-id="dtio" id="小节">小节</h2><p data-anchor-id="ox74">至此，我们讲完了基本的<strong>循环神经网络</strong>、它的训练算法：<strong>BPTT</strong>，以及在语言模型上的应用。RNN比较烧脑，相信拿下前几篇文章的读者们搞定这篇文章也不在话下吧！然而，<strong>循环神经网络</strong>这个话题并没有完结。我们在前面说到过，基本的循环神经网络存在梯度爆炸和梯度消失问题，并不能真正的处理好长距离的依赖（虽然有一些技巧可以减轻这些问题）。事实上，真正得到广泛的应用的是循环神经网络的一个变体：<strong>长短时记忆网络</strong>。它内部有一些特殊的结构，可以很好的处理长距离的依赖，我们将在下一篇文章中详细的介绍它。现在，让我们稍事休息，准备挑战更为烧脑的<strong>长短时记忆网络</strong>吧。</p><p data-anchor-id="zm3v"><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-253fd3d6688ea73e.jpg" alt="" title=""></p><div class="md-section-divider"></div><h2 data-anchor-id="6e39" id="参考资料">参考资料</h2><ol data-anchor-id="7i59">
<li><a href="http://www.wildml.com/2015/09/recurrent-neural-networks-tutorial-part-1-introduction-to-rnns/" target="_blank">RECURRENT NEURAL NETWORKS TUTORIAL</a></li>
<li><a href="http://colah.github.io/posts/2015-08-Understanding-LSTMs/" target="_blank">Understanding LSTM Networks</a></li>
<li><a href="http://karpathy.github.io/2015/05/21/rnn-effectiveness" target="_blank">The Unreasonable Effectiveness of Recurrent Neural Networks</a></li>
<li><a href="http://distill.pub/2016/augmented-rnns/" target="_blank">Attention and Augmented Recurrent Neural Networks</a></li>
<li><a href="http://www.jmlr.org/proceedings/papers/v28/pascanu13.pdf" target="_blank">On the difficulty of training recurrent neural networks, Bengio et al.</a></li>
<li><a href="http://www.fit.vutbr.cz/research/groups/speech/publi/2010/mikolov_interspeech2010_IS100722.pdf" target="_blank">Recurrent neural network based language model, Mikolov et al.</a></li>
<li><a href="https://jamesmccaffrey.wordpress.com/2011/12/17/neural-network-classification-categorical-data-softmax-activation-and-cross-entropy-error/" target="_blank">Neural Network Classification, Categorical Data, Softmax Activation, and Cross Entropy Error, McCaffrey</a></li>
</ol></div>
    <div class="remark-icons"><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="yz6y">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="07q4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top:255px;left:1350px;" data-anchor-id="id7r">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="gcom">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="hcb8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="lnzm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="b7lc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="rh2n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bifn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ah0a">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="pdef">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2b84">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="dtmk">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2pls">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="9ui4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="v19n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="c0ja">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="aqcz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5c2c">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="aoty">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="kvwy">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2qvc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="gkeg">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="n5zu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ovr4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="9p4d">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="jlbj">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="vdof">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="7oe3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="jum1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bltz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4kpk">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="db9y">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="78h0">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="91fm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2e4v">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="l5b4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="jssd">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qp28">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4xzv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="fgt4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="c2wn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="7pn8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="hjzw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bxby">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="duw7">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="da2m">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="c4ru">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6i8e">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="dj50">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ifkl">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="9vu6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="pw8b">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="alk4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="x4ve">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="jx75">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4wrm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="xvrf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ml1w">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="v0mn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="885l">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="vgoz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="gxrm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ohio">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4to7">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="g015">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="h9b8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="y7y5">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="73ed">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5y27">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3txs">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="tqfx">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="p0e6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zfra">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5r3s">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="c9zu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="mt0d">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="80lk">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6zzs">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2md8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="0p9q">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="avov">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="fels">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="tglh">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="wt0q">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="8fr9">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="mctw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="j555">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="p818">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="lheu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3omo">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="88w6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="mzvk">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="r2se">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bipj">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2v3n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qzwz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="cb0s">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="pkyc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="wi4t">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2xlu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="txsf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6dnw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="dfle">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="x0nv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="9cvp">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="s41y">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="uxjb">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="gx2x">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6fv6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="wmd2">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="wrwa">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="tkoo">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3mbn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zdj9">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="sxwc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f50z">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="mn8a">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="r4av">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="z2sb">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="mpr4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qgmg">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3l9c">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4qof">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="22da">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="rqp0">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zlqw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="mu52">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="yk0g">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qj15">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="yhhh">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="dezv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="cc4o">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3t0e">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="023a">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="h8ti">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="z8vf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="m290">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4cbn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="u32y">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ej34">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="22xz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="g5nt">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="cpiz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="0wa8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5q3y">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ay04">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="e0l9">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="iqff">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="cnga">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="lz0c">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="jrke">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qd8k">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="y2ry">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="aly3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="jfi3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6zhn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ti80">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="7n66">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="okcw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ysrn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3ljy">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="cq12">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="w9vm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5nqq">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="c3sb">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="anca">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="isy4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="erny">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="p49n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f13j">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ejtc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="oblq">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ppiv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="9qeu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="axjw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="j4lz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qf8o">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="golp">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="x4o8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="no1f">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2rn4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6ga3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4gw2">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="g7eu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qprl">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bk5i">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="rrwv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="v225">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f8i8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="uawb">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bjw7">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="kelm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="y3pl">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="kez1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f5fe">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="0rum">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="i7kc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="d8d8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="8hjo">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="yfb9">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="jvd1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5jku">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="owl3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="66ha">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="dtio">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ox74">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zm3v">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6e39">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="7i59">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div></div>
</div>

<!--in page preview buttons. -->
<div class="in-page-preview-buttons in-page-preview-buttons-full-reader">
    <ul>
        <li class="in-page-button dropdown" id="preview-toc-button" title="内容目录 Ctrl+Alt+O">
            <span class="dropdown-toggle icon-list" data-toggle="dropdown" style="color: rgba(102, 128, 153, 0.45);"></span>
            <div id="toc-list" class="dropdown-menu theme pull-right theme-white"> <!-- Add theme means this element will be changed when apply theme color. -->
                <h3>内容目录</h3>
                <hr>
                <div class="table-of-contents"><div class="toc">
<ul>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E9%9B%B6%E5%9F%BA%E7%A1%80%E5%85%A5%E9%97%A8%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A05-%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C">零基础入门深度学习(5) - 循环神经网络</a><ul>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E6%96%87%E7%AB%A0%E5%88%97%E8%A1%A8">文章列表</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%BE%80%E6%9C%9F%E5%9B%9E%E9%A1%BE">往期回顾</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B">语言模型</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E6%98%AF%E5%95%A5">循环神经网络是啥</a><ul>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%9F%BA%E6%9C%AC%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C">基本循环神经网络</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%8F%8C%E5%90%91%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C">双向循环神经网络</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E6%B7%B1%E5%BA%A6%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C">深度循环神经网络</a></li>
</ul>
</li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E8%AE%AD%E7%BB%83">循环神经网络的训练</a><ul>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E8%AE%AD%E7%BB%83%E7%AE%97%E6%B3%95bptt">循环神经网络的训练算法：BPTT</a><ul>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%89%8D%E5%90%91%E8%AE%A1%E7%AE%97">前向计算</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E8%AF%AF%E5%B7%AE%E9%A1%B9%E7%9A%84%E8%AE%A1%E7%AE%97">误差项的计算</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E6%9D%83%E9%87%8D%E6%A2%AF%E5%BA%A6%E7%9A%84%E8%AE%A1%E7%AE%97">权重梯度的计算</a></li>
</ul>
</li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#rnn%E7%9A%84%E6%A2%AF%E5%BA%A6%E7%88%86%E7%82%B8%E5%92%8C%E6%B6%88%E5%A4%B1%E9%97%AE%E9%A2%98">RNN的梯度爆炸和消失问题</a></li>
</ul>
</li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#rnn%E7%9A%84%E5%BA%94%E7%94%A8%E4%B8%BE%E4%BE%8B%E5%9F%BA%E4%BA%8Ernn%E7%9A%84%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B">RNN的应用举例——基于RNN的语言模型</a><ul>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%90%91%E9%87%8F%E5%8C%96">向量化</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#softmax%E5%B1%82">Softmax层</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B%E7%9A%84%E8%AE%AD%E7%BB%83">语言模型的训练</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E4%BA%A4%E5%8F%89%E7%86%B5%E8%AF%AF%E5%B7%AE">交叉熵误差</a></li>
</ul>
</li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#rnn%E7%9A%84%E5%AE%9E%E7%8E%B0">RNN的实现</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%B0%8F%E8%8A%82">小节</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/541458#%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99">参考资料</a></li>
</ul>
</li>
</ul>
</div>
</div>
            </div>
        </li>
    </ul>
</div>

<div id="reader-full-toolbar" class="reader-full-toolbar-shown" style="padding-top: 0;">
    <ul id="reader-full-toolbar-home" class="preview-button-row">
        <li class="preview-button-full-reader" id="preview-editor-button" title="撰写文本 Ctrl+Alt+M">
            <span class="icon-pencil" style="color: rgb(187, 187, 187);"></span>
        </li>
    </ul>
    <ul id="preview-button-row" class="preview-button-row">
        <li class="preview-button-full-reader dropdown" id="preview-list-button" title="文本列表 Ctrl+Alt+F">
            <span class="dropdown-toggle icon-reorder" data-toggle="dropdown" style="color: rgb(187, 187, 187);"></span>
            <ul id="file-list" class="dropdown-menu theme-black pull-right" role="menu" style="max-height: 652px;">
                    <li>
                    <ul class="tag-list">
                        <li class="tag-item item" tag-name="机器学习">
                            <span class="pull-left"><i class="icon-tag"></i><span class="tag-name">机器学习</span></span>
                            <span class="tag-count pull-right">7</span>
                            <div class="clearfix"></div>
                        </li>
                            
    <li class="file-item item" file-created-date="2017-02-28 00:59:57">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/626300" title="【已发布】 2017-08-29 23:41">
        <i class="icon-share-sign"></i>
        <span id="626300">零基础入门深度学习(7) - 递归神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2017-01-08 23:28:36">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/581764" title="【已发布】 2017-08-28 19:55">
        <i class="icon-share-sign"></i>
        <span id="581764">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-11-05 19:16:51">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/541458" title="【已发布】 2017-08-28 19:54">
        <i class="icon-share-sign"></i>
        <span id="541458" class="whiter-on-black">零基础入门深度学习(5) - 循环神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-10-09 20:30:46">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/485480" title="【已发布】 2017-08-28 19:53">
        <i class="icon-share-sign"></i>
        <span id="485480">零基础入门深度学习(4) - 卷积神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-08-24 21:39:25">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/476663" title="【已发布】 2017-10-17 22:25">
        <i class="icon-share-sign"></i>
        <span id="476663">零基础入门深度学习(3) - 神经网络和反向传播算法</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-26 01:44:30">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/448086" title="【已发布】 2017-08-28 19:40">
        <i class="icon-share-sign"></i>
        <span id="448086">零基础入门深度学习(2) - 线性单元和梯度下降</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-12 11:10:42">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/433855" title="【已发布】 2017-08-28 19:35">
        <i class="icon-share-sign"></i>
        <span id="433855">零基础入门深度学习(1) - 感知器</span>
        </a>
    </li>

                    </ul>
                    </li>
                    <li>
                    <ul class="tag-list">
                        <li class="tag-item item" tag-name="深度学习入门">
                            <span class="pull-left"><i class="icon-tag"></i><span class="tag-name">深度学习入门</span></span>
                            <span class="tag-count pull-right">7</span>
                            <div class="clearfix"></div>
                        </li>
                            
    <li class="file-item item" file-created-date="2017-02-28 00:59:57">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/626300" title="【已发布】 2017-08-29 23:41">
        <i class="icon-share-sign"></i>
        <span id="626300">零基础入门深度学习(7) - 递归神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2017-01-08 23:28:36">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/581764" title="【已发布】 2017-08-28 19:55">
        <i class="icon-share-sign"></i>
        <span id="581764">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-11-05 19:16:51">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/541458" title="【已发布】 2017-08-28 19:54">
        <i class="icon-share-sign"></i>
        <span id="541458" class="whiter-on-black">零基础入门深度学习(5) - 循环神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-10-09 20:30:46">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/485480" title="【已发布】 2017-08-28 19:53">
        <i class="icon-share-sign"></i>
        <span id="485480">零基础入门深度学习(4) - 卷积神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-08-24 21:39:25">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/476663" title="【已发布】 2017-10-17 22:25">
        <i class="icon-share-sign"></i>
        <span id="476663">零基础入门深度学习(3) - 神经网络和反向传播算法</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-26 01:44:30">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/448086" title="【已发布】 2017-08-28 19:40">
        <i class="icon-share-sign"></i>
        <span id="448086">零基础入门深度学习(2) - 线性单元和梯度下降</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-12 11:10:42">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/433855" title="【已发布】 2017-08-28 19:35">
        <i class="icon-share-sign"></i>
        <span id="433855">零基础入门深度学习(1) - 感知器</span>
        </a>
    </li>

                    </ul>
                    </li>
            </ul>
            <ul id="file-list-topbar" class="dropdown-menu theme-black pull-right" role="menu">
                <li id="search-file-bar">
                    <i class="icon-search icon-large"></i>
                    <input type="text" id="search-file-textbox" placeholder="搜索 hanbingtao 的文稿标题， * 显示全部">
                    <i class="icon-level-down icon-rotate-90 icon-large"></i>
                </li>
                <li id="tag-file-bar">
                    以下【标签】将用于标记这篇文稿：
                </li>
            </ul>
        </li>
        <li class="preview-button-full-reader" id="preview-theme-button" title="主题切换 Ctrl+Alt+Y">
            <span class="icon-adjust" style="color: rgb(187, 187, 187);"></span>
        </li>
        <li class="preview-button-full-reader" id="preview-fullscreen-button" title="全屏模式 F11">
            <span class="icon-fullscreen" style="color: rgb(187, 187, 187);"></span>
        </li>
        <li class="preview-button-full-reader wmd-spacer"></li>
        <li class="preview-button-full-reader dropdown" id="preview-about-button" title="关于本站">
            <span class="dropdown-toggle icon-info-sign" data-toggle="dropdown" data-hover="dropdown" data-delay="100" data-close-others="true" style="color: rgb(187, 187, 187);"></span>
            <ul id="about-menu" class="dropdown-menu theme-black pull-right" role="menu">
                <li title="下载全平台客户端"><a tabindex="-1" href="https://www.zybuluo.com/cmd" target="_blank"><i class="icon-laptop"></i>下载客户端</a></li>
                <li title="@ghosert"><a tabindex="-1" href="http://www.weibo.com/ghosert" target="_blank"><i class="icon-weibo"></i>关注开发者</a></li>
                <li title=""><a tabindex="-1" href="https://github.com/ghosert/cmd-editor/issues" target="_blank"><i class="icon-github-alt"></i>报告问题，建议</a></li>
                <li title="support@zybuluo.com"><a tabindex="-1" href="mailto:support@zybuluo.com" target="_blank"><i class="icon-envelope"></i>联系我们</a></li>
            </ul>
        </li>
    </ul>
</div>
<ul id="reader-full-toolbar-tail" class="reader-full-toolbar-tail-shown">
    <li class="preview-button-full-reader" id="preview-hidden-button" title="隐藏工具栏 Ctrl+Alt+I">
        <span class="icon-chevron-sign-right" style="color: rgb(187, 187, 187);"></span>
    </li>
</ul>






<!-- side remark, hidden when loading. -->
<div class="remark-list side-remark-hidden">
    <div class="remark-items">
    </div>
    <div class="leave-remark unselectable"><span class="icon-plus-sign-alt"></span><span>添加新批注</span></div>
    <div class="new-remark">
    <div class="remark-head"><a><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg" style="width: 32px; height: 32px;"></a></div>
    <div class="remark-author unselectable"><strong></strong></div>
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
    </div>

        <!-- clone the template $('.new-remark-reply').html() to here.-->
        <div class="remark-notice">在作者公开此批注前，只有你和作者可见。</div>
    </div>
</div>

<!-- template for new remark/reply -->
<div class="new-remark-reply side-remark-hidden">
    <div class="remark-head"><a><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg"></a></div>
    <div class="remark-author unselectable"></div>
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
    </div>
</div>

<!-- template for .remark-item/.remark-reply -->
<div class="remark-item-reply side-remark-hidden">
    <div class="remark-head"><a><img src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg"></a></div>
    <div class="remark-author unselectable"></div>
    <div class="remark-delete-link unselectable"><span class="icon-remove"></span></div> <!--This is mainly for deleting remark-reply, shown when author/remark hovering on remark-reply.-->
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-edit btn-link">修改</button>
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
        <button class="remark-delete btn-link">删除</button>
    </div>
</div>

<!-- template for remark-item-->
<div class="remark-item side-remark-hidden" data-rand-id="" data-version-id="">
    <div class="remark-published-link unselectable"><span class="icon-link icon-rotate-90"></span></div>
    <ul class="remark-options theme unselectable theme-white">
        <li class="remark-private"><span class="icon-eye-close"></span><span>私有</span></li>
        <li class="remark-public"><span class="icon-group"></span><span>公开</span></li>
        <li class="remark-delete"><span class="icon-remove"></span><span>删除</span></li>
    </ul>

    <!-- clone the template $('.remark-item-reply').html() to here.-->

    <button class="remark-reply-view-more btn-link">查看更早的 5 条回复</button>
    <div class="remark-replies">
        <!--
        <div class="remark-reply">
            clone the template $('.remark-item-reply').html() to here.
        </div>
        -->
    </div>

    <div class="leave-reply unselectable"><span>回复批注</span></div>
    <div class="new-reply">
        <!-- clone the template $('.new-remark-reply').html() to here.-->
    </div>
</div>

<!-- jiawzhang NOTICE: .remark-icons will be put to mdeditor.mako and user_note.mako, where next to .wmd-preview -->
<!-- <div class="remark-icons"></div> -->

<!-- template for remark-icon -->
<div class="remark-icon unselectable side-remark-hidden remark-icon-empty" style="display: none;">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div>


<!-- canvas, hidden always, this is used to convert svg to canvas and then convert canvas to png. -->
<canvas id="svg-canvas-image" class="editor-reader-hidden-always"></canvas>

<!-- This is the image panel to hold enlarged image/svg. -->
<div id="large-image-panel">
    <img id="large-image">
</div>


    


    <!-- Hidden Popup Modal -->
    <div id="notification-popup-window" class="modal hide fade theme theme-white" tabindex="-1" role="dialog" aria-labelledby="notification-title" aria-hidden="true">
        <div class="modal-header theme theme-white">
            <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
            <h3 id="notification-title">通知</h3>
        </div>
        <div class="modal-body theme theme-white">
            <p></p>
        </div>
        <div class="modal-footer theme theme-white">
            <button id="notification-cancel" class="btn" data-dismiss="modal" aria-hidden="true">取消</button>
            <button id="notification-confirm" class="btn btn-primary">确认</button>
        </div>
    </div>

    <!-- zybuluo's foot -->

    <script src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/288313bb.base.lib.min.js"></script>

    <script>
        Namespace('com.zybuluo.base');
        com.zybuluo.base.initData = {
            globalPromptUrl: "https://zybuluo.com/global/prompt",
        };
    </script>

    
    <!--mathjax-->
    <!--blacker: 1 below means font weight.-->
    <script type="text/x-mathjax-config;executed=true">
        MathJax.Hub.Config({ tex2jax: { inlineMath: [['$','$'], ["\\(","\\)"]], processEscapes: true }, TeX: { equationNumbers: { autoNumber: "AMS" } }, messageStyle: "none", SVG: { blacker: 1 }});
    </script>
    <script src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/mathJax.js"></script>
    <!--mathjax source code is here: https://github.com/mathjax/MathJax.-->
    <script src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/mathJax(1).js"></script>

    <script>
        Namespace('com.zybuluo.mdeditor.layout');
        com.zybuluo.mdeditor.layout.initData = {
            // '' means not logged in, otherwise the logged in username, for mdeditor.mako, this value will be reset in render.js otherwise, for user_note.mako, it's rendered by server side.
            loggedInUsername: '',
            isPageOwner: 'False' === 'True' ? true : false,
            loginComeFromUrl: 'https://zybuluo.com/login?return_to=https%3A%2F%2Fzybuluo.com%2Fhanbingtao%2Fnote%2F541458',
            noteRemarksUrl: "https://zybuluo.com/note/541458/remarks", 
            newNoteRemarkUrl: "https://zybuluo.com/note/541458/remark/new", 
            updateNoteRemarkUrl: "https://zybuluo.com/note/541458/remark/update", 
            deleteNoteRemarkUrl: "https://zybuluo.com/note/541458/remark/delete", 
            publishNoteRemarkUrl: "https://zybuluo.com/note/541458/remark/publish", 
            newNoteRemarkReplyUrl: "https://zybuluo.com/note/541458/remark_reply/new", 
            updateNoteRemarkReplyUrl: "https://zybuluo.com/note/541458/remark_reply/update", 
            deleteNoteRemarkReplyUrl: "https://zybuluo.com/note/541458/remark_reply/delete", 
        };

        // BEGIN: pace.js configuration
        window.paceOptions = {
            // disable others, enable for ajax call only,
            ajax: true,
            document: false,
            elements: false,
            eventLag: false,
        };
        // jiawzhang NOTICE: to make sure pace.js is working for any ajax call especially the jquery ajax, add 'Pace.restart()' into jquery ajax call like '$.post'
        // Originally, pace 0.5.6 doesn't support jquery ajax, see details in: https://github.com/HubSpot/pace/issues/29
        // END: pace.js configuration

    </script>

    <script src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/7a70106e.layout.lib.min.js"></script>

    <script src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/dc648f35.layout.min.js"></script><div id="medium-editor-toolbar-1" class="medium-editor-toolbar"><ul id="medium-editor-toolbar-actions" class="medium-editor-toolbar-actions clearfix"><li><button data-action="remark" class=" medium-editor-button-first"><i class="icon-comment"></i></button></li><li><button data-action="highlight" class=" medium-editor-button-last"><i class="icon-pencil"></i></button></li></ul></div>



    

    <!-- https://zybuluo.com/static/assets/mdeditor/user_note.lib.min.js -->
    <!-- -->

    <script>
        Namespace('com.zybuluo.mdeditor.user_note');
        com.zybuluo.mdeditor.user_note.initData = {
            isLoggedIn: 'False',
            mdeditorUrl: "https://zybuluo.com/mdeditor",
            passwordPassed: 'True' === 'True' ? true : false,
        };
    </script>

    <script src="./零基础入门深度学习(5) - 循环神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/6cd3112e.user_note.min.js"></script>





    
</body></html>